From 1a3983bf3199066bb2394b099ea42a0c7b16b41f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 07:07:11 +0100
Subject: add TODO

---
 bin/get-assignment | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/get-assignment b/bin/get-assignment
index d256248..f90bea2 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -15,6 +15,9 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: honor manual build order of tool-chain:
+# toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc
+
 mkdir -p "${work_dir}/package-states"
 
 hand_out_assignment() {
-- 
cgit v1.2.3-54-g00ecf


From 0ff03eeb9bcf74bb0febbde389a19cbb33253338 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 07:17:41 +0100
Subject: web-scripts/broken-packages.php: link to logs should be on reason,
 not on trial-count

---
 web-scripts/broken-packages.php | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index 268e296..d88ba4d 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -58,18 +58,25 @@ foreach ($row as $key => $val)
     );
 
     unset($reasons);
-    unset($last_log);
     $rows[$count]["trials"] = $fail_result -> num_rows;
     if ($rows[$count]["trials"] > 0) {
       while($fail_row = $fail_result->fetch_assoc()) {
-        $reasons[$fail_row["name"]] = $fail_row["name"];
-        $last_log = $fail_row["log_file"];
+        $reasons[$fail_row["name"]] = $fail_row["log_file"];
       }
     }
     if (isset($reasons)) {
       $to_print="";
-      foreach ($reasons as $reason) {
-        $to_print=$to_print.", ".$reason;
+      foreach ($reasons as $reason => $last_log) {
+        if (file_exists("/srv/http/build-logs/error/".$last_log)) {
+          $to_print= $to_print .
+            ", <a href=\"/build-logs/error/" .
+            $last_log .
+            "\">" .
+            $reason .
+            "</a>";
+        } else {
+          $to_print= $to_print . ", " . $reason;
+        }
       }
       $rows[$count]["fail_reasons"]=substr($to_print,2);
     } else {
@@ -85,10 +92,6 @@ foreach ($row as $key => $val)
     $rows[$count]["git_revision"] = $row["git_revision"];
     $rows[$count]["mod_git_revision"] = $row["mod_git_revision"];
     $rows[$count]["name"] = $row["name"];
-    if (isset($last_log))
-      $rows[$count]["print_trials"]="<a href=\"/build-logs/error/".$last_log."\">". $rows[$count]["trials"] ."</a>";
-    else
-      $rows[$count]["print_trials"]=$rows[$count]["trials"];
     if ($row["is_blocked"]=="") {
       $rows[$count]["is_blocked"]="&nbsp;";
     }
-- 
cgit v1.2.3-54-g00ecf


From 4bc90d4dbd1d0ea28ae8da27b893f01ce5ba1e32 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 09:28:56 +0100
Subject: bin/copy-to-build-support new for copying packages into build-support

---
 bin/copy-to-build-support | 48 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100755 bin/copy-to-build-support

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
new file mode 100755
index 0000000..85d73e5
--- /dev/null
+++ b/bin/copy-to-build-support
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+# copy the given package(s) into build-support
+
+# shellcheck source=conf/default.conf
+. "${0%/*}/../conf/default.conf"
+
+# shellcheck disable=SC2016
+{
+  to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture'
+  printf '%s\n' "$@" | \
+    sed -n '
+      s/.\+/\0 \0/
+      T
+      s/\.pkg\.tar\.xz$//
+      s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/
+      s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/
+      s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/
+      p
+    ' | \
+    while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do
+
+      # TODO: move packages on master mirror, too!
+
+      printf 'INSERT IGNORE INTO `binary_packages`'
+      printf ' (`repository`'
+      # shellcheck disable=SC2086
+      printf ',`%s`' ${to_copy}
+      printf ')'
+      printf ' SELECT'
+      printf ' (SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="build-support")'
+      # shellcheck disable=SC2086
+      printf ',`binary_packages`.`%s`' ${to_copy}
+      printf ' FROM `binary_packages`'
+      printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+      printf ' WHERE'
+      printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
+        'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
+        'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
+        'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
+        'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \
+        'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)"
+      printf ' `architectures`.`name`=from_base64("%s")' \
+        "$(printf '%s' "${architecture}" | base64 -w0)"
+      printf ';\n'
+    done
+} | \
+  ${mysql_command}
-- 
cgit v1.2.3-54-g00ecf


From 1b964f215c3b93c9c970137f8883b8a8c79d4319 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 09:45:03 +0100
Subject: bin/mysql-functions: mysql_find_build_assignment_loops new for
 detection of loops in build-list

---
 bin/mysql-functions | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index ec2bf4c..a1b0527 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -645,3 +645,46 @@ mysql_sanity_check() {
       "${temp_dir}/mysql-packages"
   )
 }
+
+# shellcheck disable=SC2016
+mysql_find_build_assignment_loops() {
+  new_loops=$(
+    {
+      printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`'
+      printf ' FROM `dependencies`'
+      printf ' JOIN `install_target_providers` ON `dependencies`.`depending_on`=`install_target_providers`.`install_target`'
+      printf ' JOIN `binary_packages` `packages_dependent` ON `dependencies`.`dependent`=`packages_dependent`.`id`'
+      printf ' JOIN `binary_packages` `packages_dependency` ON `install_target_providers`.`package`=`packages_dependency`.`id`'
+      printf ' JOIN `repositories` `repositories_%s` ON `packages_%s`.`repository`=`repositories_%s`.`id`' \
+        'dependent' 'dependent' 'dependent' \
+        'dependency' 'dependency' 'dependency'
+      printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
+    } | \
+      ${mysql_command} --raw --batch | \
+      tsort 2>&1 >/dev/null | \
+      sed 's/^tsort:\s*//' | \
+      {
+        loop=0
+        while read -r id; do
+          if [ "x${id}" = 'x-: input contains a loop:' ]; then
+            loop=$((loop+1))
+            continue
+          fi
+          if ! printf '%s' "${id}" | tr '\n' ' ' | grep -q '^[0-9]\+$'; then
+            >&2 printf 'ERROR: non-numeric id "%s"\n' "${id}"
+            continue
+          fi
+          printf '(%s,%s),' "${loop}" "${id}"
+        done | \
+          sed 's/,$//'
+      }
+  )
+  {
+    printf 'DELETE FROM `build_dependency_loops`;\n'
+    if [ -n "${new_loops}" ]; then
+      printf 'INSERT INTO `build_dependency_loops` (`loop`,`build_assignment`) VALUES %s;\n' \
+        "${new_loops}"
+    fi
+  } | \
+    ${mysql_command}
+}
-- 
cgit v1.2.3-54-g00ecf


From 84fc13f80e57ebea5715ac6e67502ba08efdcb02 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 10:09:34 +0100
Subject: bin/return-assignment: remove loops containing the finished package
 from database

---
 bin/return-assignment | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/bin/return-assignment b/bin/return-assignment
index cd6d30e..9de8add 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -451,6 +451,16 @@ fi
       printf '%s' "${slave}" | \
         base64 -w0
     )"
+  printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n'
+  printf 'INSERT INTO `loops_to_delete`'
+  printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`'
+  printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_dependency_loops`.`build_assignment`'
+  printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+  printf ' WHERE NOT `repositories`.`name` = "build-list";\n'
+  printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS ('
+  printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`'
+  printf ');\n'
+  printf 'DROP TABLE `loops_to_delete`;\n'
 } | \
   ${mysql_command}
 rm -f \
-- 
cgit v1.2.3-54-g00ecf


From 6f658664491b2dfdcc9a3abcbd27221f739d4851 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 10:10:48 +0100
Subject: bin/get-package-updates: keep track of dependency loops in database

---
 bin/get-package-updates | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 801aadd..e01f3dd 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -556,6 +556,9 @@ else
   rm "${work_dir}/tsort.error"
 fi
 
+# update loop list in database
+mysql_find_build_assignment_loops
+
 deletion_list_count=$(
   wc -l < \
     "${work_dir}/deletion-list.new"
-- 
cgit v1.2.3-54-g00ecf


From 7cd9746ce2914bda810f2179f7dffe03a6707665 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 10:15:00 +0100
Subject: bin/bootstrap-mysql: update operator names

---
 bin/bootstrap-mysql | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 46678f7..c643fe4 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -338,10 +338,13 @@ fi
     ' ~/.ssh/authorized_keys | \
       while read -r name key; do
         case "${name}" in
-          'nlopc'*|'rechenknecht'|'buildknecht'*)
+          'nlopc'*|'rechenknecht')
             operator='deep42thought'
           ;;
-          'eurobuild')
+          'buildknecht'*)
+            operator='deep42thought/vollzornbrot'
+          ;;
+          'eurobuild3')
             operator='abaumann'
           ;;
           *)
-- 
cgit v1.2.3-54-g00ecf


From 72bc0636528e27b4e60c4274e03820a84d2954a2 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 13:32:00 +0100
Subject: web-scripts/broken-packages.php: bugfix typo

---
 web-scripts/broken-packages.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index d88ba4d..5351bcc 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -143,7 +143,7 @@ foreach ($row as $key => $val)
     print "<td><p style=\"font-size:8px\">".$row["git_revision"]."</p></td>";
     print "<td><p style=\"font-size:8px\">".$row["mod_git_revision"]."</p></td>";
     print "<td>".$row["name"]."</td>";
-    print "<td>".$row["print_trials"]."</td>";
+    print "<td>".$row["trials"]."</td>";
     print "<td>".$row["loops"]."</td>";
 //    <td>0</td>
     print "<td>".$row["fail_reasons"]."</td>";
-- 
cgit v1.2.3-54-g00ecf


From 3658074d4134c1ab5204704a0c2900c272bea7f9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 14:02:49 +0100
Subject: web-scripts/broken-packages.php: huh, what did I do?

---
 web-scripts/broken-packages.php | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index 5351bcc..515ec22 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -27,7 +27,7 @@ $result = $mysql -> query(
     "JOIN `binary_packages` `to_be_built` ON `to_be_built`.`id` = `install_target_providers`.`package` " .
     "JOIN `repositories` ON `to_be_built`.`repository` = `repositories`.`id` " .
     "WHERE `broken_bin`.`build_assignment`=`build_assignments`.`id` ".
-    "AND `repositories`.`name`=\"community-testing\"" .
+    "AND `repositories`.`name`=\"build-list\"" .
   ") AS `dependencies_pending`," .
   "(SELECT count(*) " .
     "FROM `build_dependency_loops` " .
@@ -85,10 +85,10 @@ foreach ($row as $key => $val)
 
     $rows[$count]["loops"] = $row["loops"];
     $rows[$count]["pkgbase"] = $row["pkgbase"];
-    if ($row["dependencies_pending"]=="1")
-      $rows[$count]["pkgbase_print"] = "(" . $rows[$count]["pkgbase"] . ")";
-    else
+    if ($row["dependencies_pending"]=="0")
       $rows[$count]["pkgbase_print"] = $rows[$count]["pkgbase"];
+    else
+      $rows[$count]["pkgbase_print"] = "(" . $rows[$count]["pkgbase"] . ")";
     $rows[$count]["git_revision"] = $row["git_revision"];
     $rows[$count]["mod_git_revision"] = $row["mod_git_revision"];
     $rows[$count]["name"] = $row["name"];
-- 
cgit v1.2.3-54-g00ecf


From 465e8e28815e405832a8e76435d314e054f4c73d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 14:17:46 +0100
Subject: bin/mysql-functions: mysql_cleanup new

---
 bin/mysql-functions | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index a1b0527..0b6a5a1 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -2,7 +2,7 @@
 
 # contains functions used to access mysql db
 
-# shellcheck disable=SC2039
+# shellcheck disable=SC2016,SC2039
 
 if [ -z "${base_dir}" ]; then
   # just to make shellcheck happy
@@ -24,7 +24,7 @@ base64_encode_each() {
 
 # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
 
-# shellcheck disable=SC2016,SC2086
+# shellcheck disable=SC2086
 mysql_add_package_source() {
   local names='pkgbase git_revision mod_git_revision upstream_package_repository'
   local values;
@@ -54,7 +54,7 @@ mysql_add_package_source() {
 
 # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository
 
-# shellcheck disable=SC2016,SC2031,SC2086,SC2154
+# shellcheck disable=SC2031,SC2086,SC2154
 mysql_add_binary_package() {
   local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository'
   local name
@@ -97,7 +97,7 @@ mysql_add_binary_package() {
 
 # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel
 
-# shellcheck disable=SC2016,SC2031,SC2086,SC2154
+# shellcheck disable=SC2031,SC2086,SC2154
 mysql_show_binary_package() {
   local names='pkgname pkgver pkgrel sub_pkgrel'
   local name
@@ -153,7 +153,7 @@ mysql_show_binary_package() {
 
 # mysql_add_install_target $install_target
 
-# shellcheck disable=SC2016,2086
+# shellcheck disable=2086
 mysql_add_install_target() {
   local install_target
   install_target=$(
@@ -180,7 +180,6 @@ mysql_add_install_target() {
 # if $sub_pkgrel should be forced and the package is currently in $current_repository
 # generate the meta data of a package (dependencies, built packages, ...) in the database
 
-# shellcheck disable=SC2016
 mysql_generate_package_metadata() {
 
   ( # new shell is intentional
@@ -619,7 +618,6 @@ mysql_sanity_check() {
       sort > \
       "${temp_dir}/master-mirror-listing"
 
-    # shellcheck disable=SC2016
     {
       printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`'
       printf ' FROM `binary_packages`'
@@ -646,7 +644,6 @@ mysql_sanity_check() {
   )
 }
 
-# shellcheck disable=SC2016
 mysql_find_build_assignment_loops() {
   new_loops=$(
     {
@@ -688,3 +685,24 @@ mysql_find_build_assignment_loops() {
   } | \
     ${mysql_command}
 }
+
+# mysql_cleanup
+# clean up left overs from mysql database
+mysql_cleanup() {
+  {
+    # remove failed_builds with unbroken build_assignments
+    printf 'DELETE `failed_builds` '
+    printf 'FROM `failed_builds` '
+    printf 'JOIN `build_assignments` '
+    printf 'ON `failed_builds`.`build_assignment`=`build_assignments`.`id` '
+    printf 'WHERE NOT `build_assignment`.`is_broken`;\n'
+    # remove failed_builds w/o build_assignment
+    printf 'DELETE FROM `failed_builds` '
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `build_assignments` '
+      printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`'
+    printf ');\n'
+  } | \
+    ${mysql_command}
+}
-- 
cgit v1.2.3-54-g00ecf


From d20c019792905edc59fac5206a2e06f2f238c2af Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 14:42:12 +0100
Subject: bin/mysql-functions: fix typo

---
 bin/mysql-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 0b6a5a1..82c6fc8 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -695,7 +695,7 @@ mysql_cleanup() {
     printf 'FROM `failed_builds` '
     printf 'JOIN `build_assignments` '
     printf 'ON `failed_builds`.`build_assignment`=`build_assignments`.`id` '
-    printf 'WHERE NOT `build_assignment`.`is_broken`;\n'
+    printf 'WHERE NOT `build_assignments`.`is_broken`;\n'
     # remove failed_builds w/o build_assignment
     printf 'DELETE FROM `failed_builds` '
     printf 'WHERE NOT EXISTS '
-- 
cgit v1.2.3-54-g00ecf


From b34ca5d528f3f3aa569dffcda2d68e26849cecd1 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 15:28:47 +0100
Subject: bin/build-master-status-from-mysql: consider loops, too

---
 bin/build-master-status-from-mysql | 76 +++++++++++++++++++++++++++-----------
 1 file changed, 54 insertions(+), 22 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index a0c2b4f..b56efca 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -5,30 +5,62 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir)
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+# shellcheck disable=SC2016
+{
+  printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`'
+  printf ' FROM `package_sources`'
+  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+    'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \
+    'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
+    'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \
+    'repositories' 'binary_packages' 'repository' 'repositories'
+  printf ' WHERE `repositories`.`name`="build-list"'
+} | \
+  ${mysql_command} --batch | \
+  sed '
+    1d
+    y/\t/ /
+  ' | \
+  sort > \
+  "${tmp_dir}/build-list.mysql"
+sort "${work_dir}/build-list" > \
+  "${tmp_dir}/build-list.file"
+
+# shellcheck disable=SC2016
+{
+  printf 'SELECT `package_sources`.`pkgbase`'
+  printf ' FROM `package_sources`'
+  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+    'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
+    'build_dependency_loops' 'build_dependency_loops' 'build_assignment' 'build_assignments'
+  printf ';\n'
+} | \
+  ${mysql_command} --batch | \
+  sed '
+    1d
+    y/\t/ /
+  ' | \
+  sort > \
+  "${tmp_dir}/loops.mysql"
+find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
+  -regex '.*/loop_[0-9]\+' \
+  -exec 'cat' '{}' \; | \
+  sort > \
+  "${tmp_dir}/loops.file"
+
 {
   mysql_sanity_check || true
-  {
-    # shellcheck disable=SC2016
-    {
-      printf 'SELECT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`'
-      printf ' FROM `package_sources`'
-      printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-        'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \
-        'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
-        'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \
-        'repositories' 'binary_packages' 'repository' 'repositories'
-      printf ' WHERE `repositories`.`name`="build-list"'
-    } | \
-      ${mysql_command} --batch | \
-      sed '
-        1d
-        y/\t/ /
-        s/^/+ /
-      '
-    sed 's/^/- /' "${work_dir}/build-list"
-  } | \
-    sort -k2 -k1,1 | \
-    uniq -uf1
+  diff -u \
+    "${tmp_dir}/build-list.file" \
+    "${tmp_dir}/build-list.mysql" || \
+    true
+  diff -u \
+    "${tmp_dir}/loops.file" \
+    "${tmp_dir}/loops.mysql" || \
+    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
-- 
cgit v1.2.3-54-g00ecf


From 28b7053f6f94b02e14456f455747db75368b29d8 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 25 Jan 2018 16:50:53 +0100
Subject: bin/get-assignment: dummy test for getting next-to-built id

---
 bin/get-assignment | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/bin/get-assignment b/bin/get-assignment
index f90bea2..4f66235 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -111,6 +111,51 @@ clean_up() {
 tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
 trap clean_up EXIT
 
+# shellcheck disable=SC2016
+{
+  printf 'SELECT'
+  printf ' COALESCE('
+    printf '('
+      printf 'SELECT'
+      printf ' `build_slaves`.`currently_building`'
+      printf ' FROM `build_slaves`'
+      printf ' JOIN `build_assignments`'
+      printf ' ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
+      printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+        "$(printf '%s' "${slave}" | base64 -w0)"
+    printf '),('
+      printf 'SELECT `to_dos`.`id` FROM'
+      printf '('
+        printf 'SELECT'
+        printf ' `build_assignments`.`id`,'
+        printf '('
+          printf 'SELECT COUNT(*)'
+          printf ' FROM `failed_builds`'
+          printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
+        printf ') AS `trials`'
+        printf ' FROM `build_assignments`'
+        printf ' JOIN `binary_packages`'
+        printf ' ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+        printf ' JOIN `repositories`'
+        printf ' ON `binary_packages`.`repository`=`repositories`.`id`'
+        printf ' WHERE `repositories`.`name`="build-list"'
+        printf ' AND NOT EXISTS ('
+          printf ' SELECT *'
+          printf ' FROM `build_slaves`'
+          printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`'
+        printf ' )'
+        printf ' ORDER BY `trials`, `build_assignments`.`id`'
+        printf ' LIMIT 1'
+      printf ') AS `to_dos`'
+    printf ')'
+  printf ')'
+} | \
+  ${mysql_command} --batch --raw | \
+  sed '
+    1d
+    y/\t/ /
+  ' >&2
+
 # Check if there are any pending packages at all and if the requester
 # has already got an assignment.
 
-- 
cgit v1.2.3-54-g00ecf


From 046c477732d8bd8e1686bb01dd4b0bef881b1f72 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 26 Jan 2018 13:28:55 +0100
Subject: web-scripts/broken-packages.php: remove unneccessary "foreach"

---
 web-scripts/broken-packages.php | 2 --
 1 file changed, 2 deletions(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index 515ec22..b23b9bb 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -44,8 +44,6 @@ if ($result -> num_rows > 0) {
 
   while($row = $result->fetch_assoc()) {
 
-foreach ($row as $key => $val)
-
     $fail_result = $mysql -> query(
       "SELECT " .
       "`fail_reasons`.`name`, " .
-- 
cgit v1.2.3-54-g00ecf


From e28fa30e990687f53528b97a9849d7d00f192ab7 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 26 Jan 2018 15:22:24 +0100
Subject: bin/get-assignment: add TODO

---
 bin/get-assignment | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/get-assignment b/bin/get-assignment
index 4f66235..fc38a16 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -111,6 +111,9 @@ clean_up() {
 tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
 trap clean_up EXIT
 
+# TODO: binary package $x it is not a yet-to-be-built dependency of
+# package $y if $x and $y are built by the same build_assignment!
+
 # shellcheck disable=SC2016
 {
   printf 'SELECT'
-- 
cgit v1.2.3-54-g00ecf


From a20c7b23be889cbfaef8983e6bc14db80b369530 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 26 Jan 2018 16:25:08 +0100
Subject: bin/get-package-updates: remove done TODO

---
 bin/get-package-updates | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index e01f3dd..320cec1 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -9,8 +9,6 @@
 
 # TODO: Find out, why sometimes package updates are missed.
 
-# TODO: mark loops in mysql database
-
 # TODO: test_exclusion does not yet cooperate with the database
 
 # shellcheck disable=SC2016
-- 
cgit v1.2.3-54-g00ecf


From d5dc2eca282380293ccd86bdf7327e580573ba64 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 26 Jan 2018 16:26:17 +0100
Subject: bin/seed-build-list: remove done TODO

---
 bin/seed-build-list | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index acf25c3..cd3152d 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -5,8 +5,6 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: put new packages also in mysql database
-
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
-- 
cgit v1.2.3-54-g00ecf


From c5999ea29f2ad3bc6513aaf0f740a2ed03f10e8c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 27 Jan 2018 09:16:54 +0100
Subject: web-scripts/broken-packages.php: only show packages on the build-list

---
 web-scripts/broken-packages.php | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index b23b9bb..4980d17 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -13,7 +13,7 @@ if ($mysql->connect_error) {
 }
 
 $result = $mysql -> query(
-  "SELECT " .
+  "SELECT DISTINCT " .
   "`build_assignments`.`id`," .
   "`build_assignments`.`is_blocked`," .
   "`package_sources`.`pkgbase`," .
@@ -36,7 +36,9 @@ $result = $mysql -> query(
   "FROM `build_assignments` " .
   "JOIN `package_sources` ON `build_assignments`.`package_source` = `package_sources`.`id` " .
   "JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id` " .
-  "WHERE `build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL"
+  "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
+  "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
+  "WHERE (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL) AND `repositories`.`name`=\"build-list\""
 );
 if ($result -> num_rows > 0) {
 
-- 
cgit v1.2.3-54-g00ecf


From ab09c0383c10374ddcc587f60f36a88daabe9b42 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 28 Jan 2018 20:49:10 +0100
Subject: bin/get-package-updates: talk to irc channel if too many packages
 will be blacklisted

---
 bin/get-package-updates | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 320cec1..ce3de0a 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -564,8 +564,24 @@ deletion_list_count=$(
 if [ "${deletion_list_count}" -gt 1000 ]; then
   >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
     "${deletion_list_count}"
+  if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
+    {
+      printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
+      for repo_name in ${repo_names}; do
+        if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then
+          printf '%s: %s -> %s\n' \
+            "${repo_name}" \
+            "$(cat "${work_dir}/${repo_name}.revision")" \
+            "$(cat "${work_dir}/${repo_name}.revision.new")"
+        fi
+      done
+    } | \
+      tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
+      sponge "${irc_dir}/#archlinux-ports/in"
+  fi
   exit 3
 fi
+rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
 
 # Move the .new-files to the actual files
 
-- 
cgit v1.2.3-54-g00ecf


From a06a8eff682afb93d86fdebdef336565f26130ee Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 08:37:48 +0100
Subject: bin/common-functions: add TODO

---
 bin/common-functions | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/common-functions b/bin/common-functions
index d5b6eaa..553b9db 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -179,6 +179,8 @@ generate_package_metadata() {
     sort -u > \
     "${file_prefix}.builds"
 
+  # TODO: checkdepends should be _much_ less important!
+
   # extract "build-depends" = makedepends \cup checkdepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
   {
     {
-- 
cgit v1.2.3-54-g00ecf


From 9dc2f0416b3865298a86815d5d18b59a6a50d471 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 08:50:36 +0100
Subject: bin/build-master-status-from-mysql: look at deletion-list, too

---
 bin/build-master-status-from-mysql | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index b56efca..b1bbb98 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -29,6 +29,27 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 sort "${work_dir}/build-list" > \
   "${tmp_dir}/build-list.file"
 
+
+# shellcheck disable=SC2016
+{
+  printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
+  printf ' FROM `package_sources`'
+  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+    'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
+    'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \
+    'repositories' 'binary_packages' 'repository' 'repositories'
+  printf ' WHERE `repositories`.`name`="deletion-list";\n'
+} | \
+  ${mysql_command} --batch | \
+  sed '
+    1d
+    y/\t/ /
+  ' | \
+  sort > \
+  "${tmp_dir}/deletion-list.mysql"
+sort "${work_dir}/deletion-list" > \
+  "${tmp_dir}/deletion-list.file"
+
 # shellcheck disable=SC2016
 {
   printf 'SELECT `package_sources`.`pkgbase`'
@@ -57,6 +78,10 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
     "${tmp_dir}/build-list.file" \
     "${tmp_dir}/build-list.mysql" || \
     true
+  diff -u \
+    "${tmp_dir}/deletion-list.file" \
+    "${tmp_dir}/deletion-list.mysql" || \
+    true
   diff -u \
     "${tmp_dir}/loops.file" \
     "${tmp_dir}/loops.mysql" || \
-- 
cgit v1.2.3-54-g00ecf


From 7db1b9b1c1929e9cd6d178eed841e8f8c41b4925 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 09:26:38 +0100
Subject: bin/modify-package-state: add TODO

---
 bin/modify-package-state | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index c740d3d..326a049 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -21,6 +21,8 @@ usage() {
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
+# TODO: update database accordingly!
+
 eval set -- "$(
   getopt -o bfhntu \
     --long block \
-- 
cgit v1.2.3-54-g00ecf


From 885ff8891a08146e02f3555a25739a535b9ccc23 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 12:55:40 +0100
Subject: bin/modify-package-state: update database, too

---
 bin/modify-package-state | 85 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 84 insertions(+), 1 deletion(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 326a049..540b333 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -21,7 +21,7 @@ usage() {
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
-# TODO: update database accordingly!
+# TODO: binary packages should not be identified by revisions, but by versions
 
 eval set -- "$(
   getopt -o bfhntu \
@@ -156,6 +156,24 @@ fi
         continue
       fi
     fi
+    repo=$(
+      printf '%s' "${package##*.}" | \
+        base64 -w0
+    )
+    pkgbase="${package%.*}"
+    mod_git_rev=$(
+      printf '%s' "${package##*.}" | \
+        base64 -w0
+    )
+    pkgbase="${pkgbase%.*}"
+    git_rev=$(
+      printf '%s' "${package##*.}" | \
+        base64 -w0
+    )
+    pkgbase=$(
+      printf '%s' "${package%.*}" | \
+        base64 -w0
+    )
     if [ "${action}" = 'block' ] || \
       [ "${action}" = 'unblock' ]; then
       # these packages need to be on the build list
@@ -175,6 +193,22 @@ fi
         else
           echo "${reason}" > \
             "${work_dir}/package-states/${package}.blocked"
+          # shellcheck disable=SC2016
+          {
+            printf 'UPDATE `build_assignments`'
+            printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+            printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
+            printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \
+              "$(printf '%s' "${reason}" | base64 -w0)"
+            printf ' WHERE'
+            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
+              'pkgbase' "${pkgbase}" \
+              'git_revision' "${git_rev}" \
+              'mod_git_revision' "${mod_git_rev}"
+            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
+              "${repo}"
+          } | \
+            ${mysql_command}
           printf '%s %s\n' "${package}" "${reason}"
         fi
       ;;
@@ -183,6 +217,23 @@ fi
           mv \
             "${work_dir}/package-states/${package}.tested" \
             "${work_dir}/package-states/${package}.testing"
+          # shellcheck disable=SC2016
+          {
+            printf 'UPDATE `binary_packages`'
+            printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+              'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
+              'package_sources' 'package_sources' 'build_assignments' 'package_source' \
+              'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+            printf ' SET `binary_packages`.`has_issues`=1, `binary_packages`.`is_tested`=0'
+            printf ' WHERE'
+            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
+              'pkgbase' "${pkgbase}" \
+              'git_revision' "${git_rev}" \
+              'mod_git_revision' "${mod_git_rev}"
+            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
+              "${repo}"
+          } | \
+            ${mysql_command}
           printf '%s\n' "${package}"
         fi
       ;;
@@ -192,12 +243,44 @@ fi
           mv \
             "${work_dir}/package-states/${package}.testing" \
             "${work_dir}/package-states/${package}.tested"
+          # shellcheck disable=SC2016
+          {
+            printf 'UPDATE `binary_packages`'
+            printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+              'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
+              'package_sources' 'package_sources' 'build_assignments' 'package_source' \
+              'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+            printf ' SET `binary_packages`.`is_tested`=1'
+            printf ' WHERE'
+            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
+              'pkgbase' "${pkgbase}" \
+              'git_revision' "${git_rev}" \
+              'mod_git_revision' "${mod_git_rev}"
+            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
+              "${repo}"
+          } | \
+            ${mysql_command}
           printf '%s\n' "${package}"
         fi
       ;;
       'unblock')
         if [ -f "${work_dir}/package-states/${package}.blocked" ]; then
           rm "${work_dir}/package-states/${package}.blocked"
+          # shellcheck disable=SC2016
+          {
+            printf 'UPDATE `build_assignments`'
+            printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+            printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
+            printf ' SET `build_assignments`.`is_blocked`=NULL'
+            printf ' WHERE'
+            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
+              'pkgbase' "${pkgbase}" \
+              'git_revision' "${git_rev}" \
+              'mod_git_revision' "${mod_git_rev}"
+            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
+              "${repo}"
+          } | \
+            ${mysql_command}
           printf '%s\n' "${package}"
         fi
       ;;
-- 
cgit v1.2.3-54-g00ecf


From b2cf8dc8a879c626453879dc4d2f175d68dbfd2f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 13:42:01 +0100
Subject: bin/get-package-updates: add todo

---
 bin/get-package-updates | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index ce3de0a..d8ecd64 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -84,6 +84,8 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
+# TODO: Should new deletion-list packages be recorded in the database, too?
+
 # delete_package package
 # mark $package for deletion
 delete_package() {
-- 
cgit v1.2.3-54-g00ecf


From ce820e4c454ba94876ccd62e74d27cf24089e27d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 16:21:55 +0100
Subject: bin/get-assignment: hand out old assignments from mysql database;
 prepare handing out all assignments from database

---
 bin/get-assignment | 113 ++++++++++++++++++++++++++++++++---------------------
 1 file changed, 69 insertions(+), 44 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index fc38a16..f4ed82a 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -111,53 +111,78 @@ clean_up() {
 tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
 trap clean_up EXIT
 
-# TODO: binary package $x it is not a yet-to-be-built dependency of
-# package $y if $x and $y are built by the same build_assignment!
+# if we're building something already, hand it out (again)
+currently_building=$(
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT'
+    printf ' `package_sources`.`%s`,' \
+      'pkgbase' \
+      'git_revision' \
+      'mod_git_revision'
+    printf '`upstream_repositories`.`name`'
+    printf ' FROM `build_slaves`'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \
+      'package_sources' 'package_sources' 'build_assignments' 'package_source' \
+      'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+    printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+      "$(printf '%s' "${slave}" | base64 -w0)"
+    printf ' LIMIT 1;\n'
+  } | \
+    ${mysql_command} --batch --raw | \
+    sed '1d'
+)
+
+if [ -n "${currently_building}" ]; then
+  # shellcheck disable=SC2086
+  hand_out_assignment ${currently_building}
+fi
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT'
-  printf ' COALESCE('
+# packages with all dependencies met, ordered by number of failed trials
+next_building=$(
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT'
     printf '('
-      printf 'SELECT'
-      printf ' `build_slaves`.`currently_building`'
+      printf 'SELECT COUNT(*)'
+      printf ' FROM `failed_builds`'
+      printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
+    printf ') AS `trials`,'
+    printf '`package_sources`.`%s`,' \
+      'pkgbase' \
+      'git_revision' \
+      'mod_git_revision'
+    printf '`upstream_repositories`.`name`'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
+      'package_sources' 'package_sources' 'build_assignments' 'package_source' \
+      'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \
+      'repositories' 'repositories' 'binary_packages' 'repository'
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND NOT EXISTS ('
+      printf ' SELECT *'
       printf ' FROM `build_slaves`'
-      printf ' JOIN `build_assignments`'
-      printf ' ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
-      printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
-        "$(printf '%s' "${slave}" | base64 -w0)"
-    printf '),('
-      printf 'SELECT `to_dos`.`id` FROM'
-      printf '('
-        printf 'SELECT'
-        printf ' `build_assignments`.`id`,'
-        printf '('
-          printf 'SELECT COUNT(*)'
-          printf ' FROM `failed_builds`'
-          printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
-        printf ') AS `trials`'
-        printf ' FROM `build_assignments`'
-        printf ' JOIN `binary_packages`'
-        printf ' ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
-        printf ' JOIN `repositories`'
-        printf ' ON `binary_packages`.`repository`=`repositories`.`id`'
-        printf ' WHERE `repositories`.`name`="build-list"'
-        printf ' AND NOT EXISTS ('
-          printf ' SELECT *'
-          printf ' FROM `build_slaves`'
-          printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`'
-        printf ' )'
-        printf ' ORDER BY `trials`, `build_assignments`.`id`'
-        printf ' LIMIT 1'
-      printf ') AS `to_dos`'
-    printf ')'
-  printf ')'
-} | \
-  ${mysql_command} --batch --raw | \
-  sed '
-    1d
-    y/\t/ /
-  ' >&2
+      printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`'
+      printf ' AND `build_assignments`.`is_blocked` IS NULL'
+# TODO: and is element of loop or not exists unbuilt dependency
+    printf ' )'
+    printf ' ORDER BY `trials`, `build_assignments`.`id`'
+    printf ' LIMIT 1;\n'
+  } | \
+    ${mysql_command} --batch --raw | \
+    sed '
+      1d
+      y/\t/ /
+      s/^\S\+ //
+    '
+)
+# not ready yet!
+if [ -n "${next_building}" ]; then
+  # shellcheck disable=SC2086
+  echo hand_out_assignment ${next_building} >&2
+fi
 
 # Check if there are any pending packages at all and if the requester
 # has already got an assignment.
-- 
cgit v1.2.3-54-g00ecf


From f5474f73a5b79d85e07bf983a6a783e1e6daa0ea Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 20:08:18 +0100
Subject: bin/modify-package-state: bugfix regarding mysql

---
 bin/modify-package-state | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 540b333..b157d49 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -162,16 +162,16 @@ fi
     )
     pkgbase="${package%.*}"
     mod_git_rev=$(
-      printf '%s' "${package##*.}" | \
+      printf '%s' "${pkgbase##*.}" | \
         base64 -w0
     )
     pkgbase="${pkgbase%.*}"
     git_rev=$(
-      printf '%s' "${package##*.}" | \
+      printf '%s' "${pkgbase##*.}" | \
         base64 -w0
     )
     pkgbase=$(
-      printf '%s' "${package%.*}" | \
+      printf '%s' "${pkgbase%.*}" | \
         base64 -w0
     )
     if [ "${action}" = 'block' ] || \
@@ -207,7 +207,7 @@ fi
               'mod_git_revision' "${mod_git_rev}"
             printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
               "${repo}"
-          } | \
+          } | tee /dev/stderr | \
             ${mysql_command}
           printf '%s %s\n' "${package}" "${reason}"
         fi
-- 
cgit v1.2.3-54-g00ecf


From fd0d0003abae69107063b63ea26d6ce3feac9ecb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 29 Jan 2018 23:08:33 +0100
Subject: web-scripts/broken-packages.php: fix link

---
 web-scripts/broken-packages.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index 4980d17..b506cec 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -4,7 +4,7 @@
 <link rel="stylesheet" type="text/css" href="/static/style.css">
 </head>
 <body>
-<a href="build-logs/">build logs</a><br>
+<a href="/build-logs/">build logs</a><br>
 <?php
 
 $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
-- 
cgit v1.2.3-54-g00ecf


From 6c9b41e2ce003a1920daa2ca7b43365d99fa29dc Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 30 Jan 2018 10:22:30 +0100
Subject: bin/get-package-updates: fix typo in comment

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index d8ecd64..069277a 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -289,7 +289,7 @@ done | \
       # shellcheck disable=SC2016
       {
         # delete old build assignment and associated binary packages
-        # which are not yet build or on the deletion list
+        # which are not yet built or on the deletion list
         printf 'DELETE `build_assignments`,`binary_packages`'
         printf ' FROM `binary_packages`'
         printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-- 
cgit v1.2.3-54-g00ecf


From f1a5d0d8305185d40d3c465e0a2b141bcdb3ddbb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 30 Jan 2018 10:43:43 +0100
Subject: bin/get-assignment: myqsl query finished

---
 bin/get-assignment | 41 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index f4ed82a..270367e 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -144,6 +144,11 @@ next_building=$(
   # shellcheck disable=SC2016
   {
     printf 'SELECT'
+    printf ' `package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \
+      "$(
+        printf '%s' "$1" | \
+          base64 -w0
+      )"
     printf '('
       printf 'SELECT COUNT(*)'
       printf ' FROM `failed_builds`'
@@ -165,17 +170,43 @@ next_building=$(
       printf ' SELECT *'
       printf ' FROM `build_slaves`'
       printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`'
-      printf ' AND `build_assignments`.`is_blocked` IS NULL'
-# TODO: and is element of loop or not exists unbuilt dependency
-    printf ' )'
-    printf ' ORDER BY `trials`, `build_assignments`.`id`'
+    printf ') AND ('
+      printf '`build_assignments`.`is_blocked` IS NULL'
+      printf ' OR'
+      printf ' `package_sources`.`pkgbase`=from_base64("%s")' \
+        "$(
+          printf '%s' "$1" | \
+            base64 -w0
+        )"
+    printf ') AND ('
+      printf 'EXISTS( '
+        printf 'SELECT * FROM `build_dependency_loops`'
+        printf ' WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`'
+# TODO: ignore loops which contain a build_assignment currently being handed out (?)
+      printf ') OR'
+      printf ' NOT EXISTS ('
+        printf 'SELECT * FROM `binary_packages` as `to_dos`'
+        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \
+          'repositories` AS `to_do_repos' 'to_do_repos'  'id'           'to_dos'                   'repository' \
+          'dependencies'                  'to_dos'       'id'           'dependencies'             'dependent' \
+          'install_target_providers'      'dependencies' 'depending_on' 'install_target_providers' 'install_target' \
+          'binary_packages` AS `bin_deps' 'bin_deps'     'id'           'install_target_providers' 'package' \
+          'repositories` AS `dep_repos'   'dep_repos'    'id'           'bin_deps'                 'repository'
+        printf ' WHERE'
+        printf ' `%s`.`name`="build-list" AND' \
+          'dep_repos' 'to_do_repos'
+        printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND'
+        printf ' `to_dos`.`build_assignment`=`build_assignments`.`id`'
+      printf ')'
+    printf ')'
+    printf ' ORDER BY `requested` DESC, `trials`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
   } | \
     ${mysql_command} --batch --raw | \
     sed '
       1d
       y/\t/ /
-      s/^\S\+ //
+      s/^\S\+ \S\+ //
     '
 )
 # not ready yet!
-- 
cgit v1.2.3-54-g00ecf


From 4e4e93319ba811c477864b89224cca55b9f8bad5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 30 Jan 2018 11:33:50 +0100
Subject: web-scripts/broken-packages.php: do not count split packages
 depending on each other as dependencies

---
 web-scripts/broken-packages.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index b506cec..4a42d92 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -27,7 +27,8 @@ $result = $mysql -> query(
     "JOIN `binary_packages` `to_be_built` ON `to_be_built`.`id` = `install_target_providers`.`package` " .
     "JOIN `repositories` ON `to_be_built`.`repository` = `repositories`.`id` " .
     "WHERE `broken_bin`.`build_assignment`=`build_assignments`.`id` ".
-    "AND `repositories`.`name`=\"build-list\"" .
+    "AND `repositories`.`name`=\"build-list\" " .
+    "AND `to_be_built`.`build_assignment`!=`build_assignments`.`id`" .
   ") AS `dependencies_pending`," .
   "(SELECT count(*) " .
     "FROM `build_dependency_loops` " .
-- 
cgit v1.2.3-54-g00ecf


From 1bef73485855314342c66d9bf43e3c71ffac6cbb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 30 Jan 2018 14:00:25 +0100
Subject: bin/bootstrap-mysql: new dependency types

---
 bin/bootstrap-mysql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index c643fe4..725b211 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -421,6 +421,8 @@ fi
   {
     printf '\n  ("%s",%s,%s),' \
       'build' '1' '0' \
+      'check' '0' '0' \
+      'link' '0' '1' \
       'run' '1' '1'
     printf ';\n'
   } | \
-- 
cgit v1.2.3-54-g00ecf


From a186d4ffccab0d255e890dcaf563e2652c5b176e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 30 Jan 2018 14:01:28 +0100
Subject: bin/get-assignment: clarify comment

---
 bin/get-assignment | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 270367e..298ea78 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -139,7 +139,9 @@ if [ -n "${currently_building}" ]; then
   hand_out_assignment ${currently_building}
 fi
 
-# packages with all dependencies met, ordered by number of failed trials
+# a package with all dependencies met or which is part of a loop,
+#   that we requested or
+#   which has been built the least times
 next_building=$(
   # shellcheck disable=SC2016
   {
-- 
cgit v1.2.3-54-g00ecf


From d5cca713716f5ea3a8ee843ebd9f128878045465 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 30 Jan 2018 14:18:31 +0100
Subject: bin/get-assignment: hand out packages according to mysql

---
 bin/get-assignment | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 298ea78..0e37e14 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -211,10 +211,9 @@ next_building=$(
       s/^\S\+ \S\+ //
     '
 )
-# not ready yet!
 if [ -n "${next_building}" ]; then
   # shellcheck disable=SC2086
-  echo hand_out_assignment ${next_building} >&2
+  hand_out_assignment ${next_building}
 fi
 
 # Check if there are any pending packages at all and if the requester
-- 
cgit v1.2.3-54-g00ecf


From 70074e1bdf42534b3a1cf7c23b714104e59d9ae5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 30 Jan 2018 14:56:11 +0100
Subject: bin/get-assignment: extract some mysql statements into functions in
 bin/mysql-functions

---
 bin/get-assignment  | 52 +++++++++++-----------------------------------------
 bin/mysql-functions | 45 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 41 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 0e37e14..ec8dad2 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -115,17 +115,10 @@ trap clean_up EXIT
 currently_building=$(
   # shellcheck disable=SC2016
   {
-    printf 'SELECT'
-    printf ' `package_sources`.`%s`,' \
-      'pkgbase' \
-      'git_revision' \
-      'mod_git_revision'
-    printf '`upstream_repositories`.`name`'
-    printf ' FROM `build_slaves`'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \
-      'package_sources' 'package_sources' 'build_assignments' 'package_source' \
-      'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+    printf 'SELECT '
+    mysql_query_select_pkgbase_and_revision
+    printf ' JOIN `build_slaves`'
+    printf ' ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
     printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
       "$(printf '%s' "${slave}" | base64 -w0)"
     printf ' LIMIT 1;\n'
@@ -145,8 +138,8 @@ fi
 next_building=$(
   # shellcheck disable=SC2016
   {
-    printf 'SELECT'
-    printf ' `package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \
+    printf 'SELECT '
+    printf '`package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \
       "$(
         printf '%s' "$1" | \
           base64 -w0
@@ -156,16 +149,9 @@ next_building=$(
       printf ' FROM `failed_builds`'
       printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
     printf ') AS `trials`,'
-    printf '`package_sources`.`%s`,' \
-      'pkgbase' \
-      'git_revision' \
-      'mod_git_revision'
-    printf '`upstream_repositories`.`name`'
-    printf ' FROM `binary_packages`'
+    mysql_query_select_pkgbase_and_revision
     printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
-      'package_sources' 'package_sources' 'build_assignments' 'package_source' \
-      'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \
+      'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \
       'repositories' 'repositories' 'binary_packages' 'repository'
     printf ' WHERE `repositories`.`name`="build-list"'
     printf ' AND NOT EXISTS ('
@@ -181,25 +167,9 @@ next_building=$(
             base64 -w0
         )"
     printf ') AND ('
-      printf 'EXISTS( '
-        printf 'SELECT * FROM `build_dependency_loops`'
-        printf ' WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`'
-# TODO: ignore loops which contain a build_assignment currently being handed out (?)
-      printf ') OR'
-      printf ' NOT EXISTS ('
-        printf 'SELECT * FROM `binary_packages` as `to_dos`'
-        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \
-          'repositories` AS `to_do_repos' 'to_do_repos'  'id'           'to_dos'                   'repository' \
-          'dependencies'                  'to_dos'       'id'           'dependencies'             'dependent' \
-          'install_target_providers'      'dependencies' 'depending_on' 'install_target_providers' 'install_target' \
-          'binary_packages` AS `bin_deps' 'bin_deps'     'id'           'install_target_providers' 'package' \
-          'repositories` AS `dep_repos'   'dep_repos'    'id'           'bin_deps'                 'repository'
-        printf ' WHERE'
-        printf ' `%s`.`name`="build-list" AND' \
-          'dep_repos' 'to_do_repos'
-        printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND'
-        printf ' `to_dos`.`build_assignment`=`build_assignments`.`id`'
-      printf ')'
+      mysql_query_is_part_of_loop '`build_assignments`.`id`'
+      printf ' OR NOT '
+      mysql_query_has_pending_dependencies '`build_assignments`.`id`'
     printf ')'
     printf ' ORDER BY `requested` DESC, `trials`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
diff --git a/bin/mysql-functions b/bin/mysql-functions
index 82c6fc8..44ef851 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -706,3 +706,48 @@ mysql_cleanup() {
   } | \
     ${mysql_command}
 }
+
+# mysql_query_has_pending_dependencies `build_assignment`.`id`
+# print a mysql query giving wether dependencies are pending
+mysql_query_has_pending_dependencies() {
+  printf 'EXISTS ('
+    printf 'SELECT * FROM `binary_packages` as `to_dos`'
+    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \
+      'repositories` AS `to_do_repos' 'to_do_repos'  'id'           'to_dos'                   'repository' \
+      'dependencies'                  'to_dos'       'id'           'dependencies'             'dependent' \
+      'install_target_providers'      'dependencies' 'depending_on' 'install_target_providers' 'install_target' \
+      'binary_packages` AS `bin_deps' 'bin_deps'     'id'           'install_target_providers' 'package' \
+      'repositories` AS `dep_repos'   'dep_repos'    'id'           'bin_deps'                 'repository'
+    printf ' WHERE'
+    printf ' `%s`.`name`="build-list" AND' \
+      'dep_repos' 'to_do_repos'
+    printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND'
+    printf ' `to_dos`.`build_assignment`=%s' \
+      "$1"
+  printf ')'
+}
+
+# mysql_query_is_part_of_loop `build_assignment`.`id`
+# print a mysql query giving wether the package is part of a loop
+mysql_query_is_part_of_loop() {
+  printf 'EXISTS ('
+    printf 'SELECT * FROM `build_dependency_loops`'
+    printf ' WHERE `build_dependency_loops`.`build_assignment`=%s' \
+      "$1"
+  printf ')'
+}
+
+# mysql_query_select_pkgbase_and_revision
+# print the part of a mysql query giving:
+# pkgbase git_revision mod_git_revision upstream_package_repository
+mysql_query_select_pkgbase_and_revision() {
+  printf '`package_sources`.`%s`,' \
+    'pkgbase' \
+    'git_revision' \
+    'mod_git_revision'
+  printf '`upstream_repositories`.`name`'
+  printf ' FROM `build_assignments`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'package_sources' 'package_sources' 'build_assignments' 'package_source' \
+    'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+}
-- 
cgit v1.2.3-54-g00ecf


From c2ce72f0fb3218ba977e257508dbe8d19e811e27 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 10:39:20 +0100
Subject: bin/bootstrap-mysql: more columns in `package_sources`

---
 bin/bootstrap-mysql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 725b211..6c16a1a 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -99,6 +99,8 @@ if [ ! "$1" = 'slim' ]; then
       ' git_revision VARCHAR(40)' \
       ' mod_git_revision VARCHAR(40)' \
       ' upstream_package_repository SMALLINT :upstream_repositories' \
+      ' uses_upstream BIT' \
+      ' uses_modification BIT' \
       ' UNIQUE content pkgbase git_revision mod_git_revision' \
       'upstream_repositories SMALLINT' \
       ' name VARCHAR(64)' \
-- 
cgit v1.2.3-54-g00ecf


From c310a56816bf44439b1ecc79d8e3b2cd099102a0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 10:40:35 +0100
Subject: bin/mysql-functions: mysql_add_package_source should fill
 uses_upstream and uses_modification, too

---
 bin/mysql-functions | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 44ef851..ac39141 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -27,7 +27,25 @@ base64_encode_each() {
 # shellcheck disable=SC2086
 mysql_add_package_source() {
   local names='pkgbase git_revision mod_git_revision upstream_package_repository'
-  local values;
+  local values
+  local uses_upstream
+  local uses_modification
+
+  if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then
+    uses_modification=1
+  else
+    uses_modification=0
+  fi
+  uses_upstream=0
+  for values in ${repo_names}; do
+    if eval 'git -C "${repo_paths__'"${values}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
+      tar -t 2>/dev/null | \
+      sed 's,-x86_64/,-any/,' | \
+      grep -qFx "$1/repos/$4-any/PKGBUILD"; then
+      uses_upstream=1
+    fi
+  done
+
   for _ in ${names}; do
     values="${values}$(
       printf '%s' "$1" | \
@@ -41,14 +59,15 @@ mysql_add_package_source() {
     printf 'INSERT IGNORE INTO package_sources'
     printf ' ('
     printf '`%s`, ' ${names}
+    printf '`uses_upstream`,`uses_modification`'
     printf ') SELECT'
     printf ' from_base64("%s"), ' ${values% *}
-    printf ' `upstream_repositories`.`id`'
+    printf ' `upstream_repositories`.`id`,%s,%s' \
+      ${uses_upstream} ${uses_modification}
     printf ' FROM `upstream_repositories`'
     printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \
       "${values##* }"
   } | \
-    sed 's|, )|)|g' | \
     ${mysql_command}
 }
 
-- 
cgit v1.2.3-54-g00ecf


From 3c733778909fd15af6308fda7f82fa995c6660b4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 11:16:52 +0100
Subject: web-scripts/broken-packages.php: add links to git repositories on
 respective file and commit (if relevant)

---
 web-scripts/broken-packages.php | 40 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index 4a42d92..3468681 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -19,7 +19,11 @@ $result = $mysql -> query(
   "`package_sources`.`pkgbase`," .
   "`package_sources`.`git_revision`," .
   "`package_sources`.`mod_git_revision`," .
-  "`upstream_repositories`.`name`," .
+  "`package_sources`.`uses_upstream`," .
+  "`package_sources`.`uses_modification`," .
+  "`upstream_repositories`.`name` AS `package_repository`," .
+  "`git_repositories`.`name` AS `git_repository`," .
+  "`architectures`.`name` AS `arch`," .
   "EXISTS (SELECT * " .
     "FROM `binary_packages` `broken_bin` " .
     "JOIN `dependencies` ON `dependencies`.`dependent` = `broken_bin`.`id` " .
@@ -35,8 +39,10 @@ $result = $mysql -> query(
     "WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`" .
   ") AS `loops` " .
   "FROM `build_assignments` " .
+  "JOIN `architectures` ON `build_assignments`.`architecture` = `architectures`.`id` " .
   "JOIN `package_sources` ON `build_assignments`.`package_source` = `package_sources`.`id` " .
   "JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id` " .
+  "JOIN `git_repositories` ON `upstream_repositories`.`git_repository`=`git_repositories`.`id` " .
   "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
   "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
   "WHERE (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL) AND `repositories`.`name`=\"build-list\""
@@ -90,9 +96,33 @@ if ($result -> num_rows > 0) {
       $rows[$count]["pkgbase_print"] = $rows[$count]["pkgbase"];
     else
       $rows[$count]["pkgbase_print"] = "(" . $rows[$count]["pkgbase"] . ")";
-    $rows[$count]["git_revision"] = $row["git_revision"];
-    $rows[$count]["mod_git_revision"] = $row["mod_git_revision"];
-    $rows[$count]["name"] = $row["name"];
+    if ($row["uses_upstream"]) {
+      $rows[$count]["git_revision"] =
+        "<a href=\"https://git.archlinux.org/svntogit/" .
+        $row["git_repository"] . ".git/tree/" .
+        $row["pkgbase"] . "/repos/" .
+        $row["package_repository"] . "-";
+      if ($row["arch"]=="any")
+        $rows[$count]["git_revision"] =
+          $rows[$count]["git_revision"] . "any";
+      else
+        $rows[$count]["git_revision"] =
+          $rows[$count]["git_revision"] . "x86_64";
+      $rows[$count]["git_revision"] =
+        $rows[$count]["git_revision"] . "/\">" .
+        $row["git_revision"] . "</a>";
+    } else
+      $rows[$count]["git_revision"] = $row["git_revision"];
+    if ($row["uses_modification"])
+      $rows[$count]["mod_git_revision"] =
+        "<a href=\"https://github.com/archlinux32/packages/tree/" .
+        $row["mod_git_revision"] . "/" .
+        $row["package_repository"] . "/" .
+        $row["pkgbase"] . "\">" .
+        $row["mod_git_revision"] . "</a>";
+    else
+      $rows[$count]["mod_git_revision"] = $row["mod_git_revision"];
+    $rows[$count]["package_repository"] = $row["package_repository"];
     if ($row["is_blocked"]=="") {
       $rows[$count]["is_blocked"]="&nbsp;";
     }
@@ -143,7 +173,7 @@ if ($result -> num_rows > 0) {
     print "<td><a href=\"/graphs/".$row["pkgbase"].".png\">".$row["pkgbase_print"]."</a></td>";
     print "<td><p style=\"font-size:8px\">".$row["git_revision"]."</p></td>";
     print "<td><p style=\"font-size:8px\">".$row["mod_git_revision"]."</p></td>";
-    print "<td>".$row["name"]."</td>";
+    print "<td>".$row["package_repository"]."</td>";
     print "<td>".$row["trials"]."</td>";
     print "<td>".$row["loops"]."</td>";
 //    <td>0</td>
-- 
cgit v1.2.3-54-g00ecf


From 90a901dd785c216c4000b23352f2b0bf278ae231 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 11:58:56 +0100
Subject: bin/get-assignment: hand out all-dep-met packages first and loops
 later

---
 bin/get-assignment | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index ec8dad2..27b622b 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -149,6 +149,8 @@ next_building=$(
       printf ' FROM `failed_builds`'
       printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
     printf ') AS `trials`,'
+    mysql_query_is_part_of_loop '`build_assignments`.`id`'
+    printf ' AS `part_of_loop`,'
     mysql_query_select_pkgbase_and_revision
     printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
       'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \
@@ -171,14 +173,14 @@ next_building=$(
       printf ' OR NOT '
       mysql_query_has_pending_dependencies '`build_assignments`.`id`'
     printf ')'
-    printf ' ORDER BY `requested` DESC, `trials`, `build_assignments`.`id`'
+    printf ' ORDER BY `requested` DESC, `trials`, `part_of_loop`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
   } | \
     ${mysql_command} --batch --raw | \
     sed '
       1d
       y/\t/ /
-      s/^\S\+ \S\+ //
+      s/^\(\S\+ \)\{3\}//
     '
 )
 if [ -n "${next_building}" ]; then
-- 
cgit v1.2.3-54-g00ecf


From b72d5fcdc6ea158edef7b3127d1edd00882ae992 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 12:01:21 +0100
Subject: bin/get-assignment: rely on database - kick out state file stuff :-)

---
 bin/get-assignment | 153 +++++++----------------------------------------------
 1 file changed, 20 insertions(+), 133 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 27b622b..48e8d53 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -188,140 +188,27 @@ if [ -n "${next_building}" ]; then
   hand_out_assignment ${next_building}
 fi
 
-# Check if there are any pending packages at all and if the requester
-# has already got an assignment.
-
-pending_packages=false
-
-while read -r package git_revision mod_git_revision repository; do
-
-  generate_package_metadata "${package}.${git_revision}.${mod_git_revision}.${repository}"
-
-  if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then
-    # has this slave already got or does he prefer this assignment?
-    # note, that the dependencies are met, because this package is already locked
-    if grep -qxF "${slave}" "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" || \
-      printf ',%s,' "$1" | \
-        grep -qF ",${package},"; then
-      hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-    fi
-  elif [ ! -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
-    pending_packages=true
-  fi
-
-done < "${work_dir}/build-list"
+# Check if there are any pending packages at all
+count_pending=$(
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT count(*)'
+    printf ' FROM `build_assignments`'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \
+      'repositories' 'repositories' 'binary_packages' 'repository'
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_assignments`.`is_blocked` IS NULL'
+    printf ';\n'
+  } | \
+    ${mysql_command} --batch --raw | \
+    sed '1d'
+)
 
-if ! ${pending_packages}; then
+if [ "${count_pending}" -eq 0 ]; then
   >&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages'
   exit 3
+else
+  >&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready'
+  exit 2
 fi
-
-# Find first package of build-list whose "dependencies" are all met
-# 1st: prefered packages on the build list which have all dependencies met
-# 2nd: unbroken packages on the build list which have all dependencies met
-# 3rd: unbroken packages breaking a loop
-# 4th: broken packages which have all dependencies met or break a loop
-
-for iteration in 'prefered' 'fresh' 'loops' 'broken'; do
-
-  case "${iteration}" in
-    'prefered')
-      hand_out_blocked=true
-      hand_out_broken=true
-      hand_out_loop=false
-      echo "$1" | \
-        tr ',' '\n' | \
-        sort -u > \
-        "${tmp_dir}/hand-out-only-these-packages"
-    ;;
-    'fresh')
-      hand_out_blocked=false
-      hand_out_broken=false
-      hand_out_loop=false
-      {
-        cat "${work_dir}/build-list"
-        find "${work_dir}/package-states" -name '*.broken' -printf '%f\n' | \
-          sed '
-            s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
-            p
-          '
-      } | \
-        sort -k1,1 -k2 | \
-        uniq -u | \
-        cut -d' ' -f1 | \
-        uniq > \
-        "${tmp_dir}/hand-out-only-these-packages"
-    ;;
-    'loops')
-      hand_out_blocked=false
-      hand_out_broken=false
-      hand_out_loop=true
-      find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-        -regex '.*/loop_[0-9]\+' \
-        -exec cat {} \; | \
-        sort -u > \
-        "${tmp_dir}/hand-out-only-these-packages"
-    ;;
-    'broken')
-      hand_out_blocked=false
-      hand_out_broken=true
-      hand_out_loop=true
-      cut -d' ' -f1 < \
-        "${work_dir}/build-list" | \
-        sort -u > \
-        "${tmp_dir}/hand-out-only-these-packages"
-    ;;
-  esac
-
-  cat -n "${work_dir}/build-list" | \
-    sort -k2,2 | \
-    join -1 1 -2 2 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/hand-out-only-these-packages" - | \
-    sort -k1,1 | \
-    sed 's|^\s*\S\+\s\+||' > \
-    "${tmp_dir}/try-to-hand-out-these-packages"
-
-  while read -r package git_revision mod_git_revision repository; do
-
-    # package locked? (we have taken care of those in the previous loop)
-    if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then
-      continue
-    fi
-
-    # package blocked?
-    if ! ${hand_out_blocked} &&
-      [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
-      continue
-    fi
-
-    # package broken?
-    if ! ${hand_out_broken} && \
-      [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then
-      continue
-    fi
-
-    # dependencies met?
-    if [ -n "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
-      # do we hand out looped packages?
-      if ! ${hand_out_loop}; then
-        continue
-      fi
-      # is it a looped package?
-      if ! find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-        -regex '.*/loop_[0-9]\+' \
-        -exec cat {} \; | \
-        grep -qxF "${package}"; then
-        continue
-      fi
-    fi
-
-    hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
-  done < \
-    "${tmp_dir}/try-to-hand-out-these-packages"
-
-done
-
-# Remove the lock file
-
->&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready'
-exit 2
-- 
cgit v1.2.3-54-g00ecf


From 2a359e3274d45123bc52f7efa58f3459df35f05c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 12:12:19 +0100
Subject: bin/mysql-functions: bugfix

---
 bin/mysql-functions | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index ac39141..f3dcc5b 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -30,6 +30,7 @@ mysql_add_package_source() {
   local values
   local uses_upstream
   local uses_modification
+  local repo
 
   if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then
     uses_modification=1
@@ -37,7 +38,7 @@ mysql_add_package_source() {
     uses_modification=0
   fi
   uses_upstream=0
-  for values in ${repo_names}; do
+  for repo in ${repo_names}; do
     if eval 'git -C "${repo_paths__'"${values}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
       tar -t 2>/dev/null | \
       sed 's,-x86_64/,-any/,' | \
-- 
cgit v1.2.3-54-g00ecf


From f06d489ccfae3607d0b5bf442b38fe1ae81b62d6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 13:44:37 +0100
Subject: bin/get-package-updates: read lib32-* packages from database

---
 bin/get-package-updates | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 069277a..55efee4 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -343,9 +343,14 @@ black_listed_new=$(
     if [ -n "${test_exclusion}" ]; then
       echo "${test_exclusion}"
     fi
-    find "${work_dir}/package-infos" -maxdepth 1 -name 'lib32-*' -printf '%f\n' | \
-      sed 's|\(\.[^.]\+\)\{4\}$||' | \
-      uniq
+    # shellcheck disable=SC2016
+    {
+      printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
+      printf ' FROM `package_sources`'
+      printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '1d'
   } | \
     sort -u
 )
-- 
cgit v1.2.3-54-g00ecf


From b36b0ca61efc66dc0bc4037f4fb5851d0d4d5059 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 15:10:07 +0100
Subject: bin/bootstrap-mysql: set correct bug tracker categories

---
 bin/bootstrap-mysql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 6c16a1a..193b0eb 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -377,11 +377,11 @@ fi
   printf 'INSERT IGNORE INTO `repository_stabilities` (`name`,`bugtracker_category`) VALUES'
   {
     printf '\n  ("%s",%s),' \
-      'stable' '"Packages:Stable"' \
-      'testing' '"Packages:Testing"' \
+      'stable' '"Packages: Stable"' \
+      'testing' '"Packages: Testing"' \
       'staging' 'NULL' \
       'standalone' 'NULL' \
-      'unbuilt' '"Packages:Build-list"' \
+      'unbuilt' '"Packages: Build-list"' \
       'forbidden' 'NULL'
     printf ';\n'
   } | \
-- 
cgit v1.2.3-54-g00ecf


From 4b729829aa589aed54d2811ad0190a00b6914e46 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 15:17:53 +0100
Subject: bin/common-functions:
 extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name new

---
 bin/common-functions | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/bin/common-functions b/bin/common-functions
index 553b9db..03a177b 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -1348,3 +1348,27 @@ trigger_mirror_refreshs() {
     screen -S trigger-mirror-update -d -m curl -L "${trigger_url}"
   done
 }
+
+# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name
+extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() {
+  pkgname="$1"
+  pkgname="${pkgname%.pkg.tar.xz}"
+  arch="${pkgname##*-}"
+  pkgname="${pkgname%-*}"
+  sub_pkgrel="${pkgname##*-}"
+  pkgname="${pkgname%-*}"
+  pkgrel="${sub_pkgrel%.*}"
+  if [ "${pkgrel}" = "${sub_pkgrel}" ]; then
+    sub_pkgrel='0'
+  else
+    sub_pkgrel="${sub_pkgrel##*.}"
+  fi
+  epoch="${pkgname##*-}"
+  pkgname="${pkgname%-*}"
+  pkgver="${epoch#*:}"
+  if [ "${pkgver}" = "${epoch}" ]; then
+    epoch='0'
+  else
+    epoch="${epoch%%:*}"
+  fi
+}
-- 
cgit v1.2.3-54-g00ecf


From b3ae76ed20e9aaaf6e45e3ac8146898d895b65b6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 15:37:44 +0100
Subject: bin/modify-package-state: solely rely on database

---
 bin/modify-package-state | 284 ++++++++++++++++++++---------------------------
 1 file changed, 118 insertions(+), 166 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index b157d49..477f3c3 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+# shellcheck disable=SC2039
+
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
@@ -97,12 +99,6 @@ if [ $# -ne 1 ]; then
   usage
 fi
 
-clean_up() {
-  rm -rf --one-file-system "${tmp_dir}"
-}
-tmp_dir=$(mktemp -d 'tmp.modify-package-state.XXXXXXXXXX' --tmpdir)
-trap clean_up EXIT
-
 input_file="$1"
 if ${report}; then
   if ! [ -w "${input_file}" ]; then
@@ -113,10 +109,9 @@ if ${report}; then
   fi
   move_output() {
     cat "${output_file}" > "${input_file}"
-    clean_up
+    rm -f "${output_file}"
   }
-  output_file="${tmp_dir}/output-file"
-  touch "${output_file}"
+  output_file=$(mktemp 'tmp.modify-package-state.XXXXXXXXXX')
   trap 'move_output' EXIT
 else
   output_file='/dev/null'
@@ -135,163 +130,120 @@ if ! flock -s -n 9; then
   exit
 fi
 
-if [ "${action}" = 'tested' ]; then
-  receive_buglist 'Testing' > \
-    "${tmp_dir}/package-bug-titles"
-fi
+# TODO: handle bug reports, too
+# receive_buglist 'Testing'
 
-{
-  err=0
-  while read -r package reason; do
-    if echo "${package}" | \
-      grep -q '\.pkg\.tar\.xz$'; then
-      package=$(
-        find "${work_dir}/package-states" -maxdepth 1 \( -name '*.tested' -o -name '*.testing' \) \
-          -exec grep -qxF "${package}" '{}' \; \
-          -printf '%f\n' | \
-          sed 's|\.[^.]\+$||' | \
-          sort -u
+while read -r package reason; do
+  case "${action}" in
+    'faulty'|'tested')
+      # we expect a binary package identifier (pkgname-epoch:pkgver-pkgrel.sub_pkgrel-arch[".pkg.tar.xz"])
+      # and we will only operate on packages in "testing" repositories
+      # shellcheck disable=SC2016
+      combiner=$(
+        printf '`binary_packages`'
+        printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+          'repositories'           'repositories'           'binary_packages' 'repository' \
+          'repository_stabilities' 'repository_stabilities' 'repositories'    'stability' \
+          'architectures'          'architectures'          'binary_packages' 'architecture'
       )
-      if [ -z "${package}" ]; then
-        continue
-      fi
-    fi
-    repo=$(
-      printf '%s' "${package##*.}" | \
-        base64 -w0
-    )
-    pkgbase="${package%.*}"
-    mod_git_rev=$(
-      printf '%s' "${pkgbase##*.}" | \
-        base64 -w0
-    )
-    pkgbase="${pkgbase%.*}"
-    git_rev=$(
-      printf '%s' "${pkgbase##*.}" | \
-        base64 -w0
-    )
-    pkgbase=$(
-      printf '%s' "${pkgbase%.*}" | \
-        base64 -w0
-    )
-    if [ "${action}" = 'block' ] || \
-      [ "${action}" = 'unblock' ]; then
-      # these packages need to be on the build list
-      if ! tr ' ' '.' < \
-        "${work_dir}/build-list" | \
-        grep -qxF "${package}"; then
-        >&2 printf 'Package "%s" is not on the build-list.\n' "${package}"
-        err=2
-        continue
+      # shellcheck disable=SC2016
+      selector=$(
+        local pkgname
+        local epoch
+        local pkgver
+        local pkgrel
+        local sub_pkgrel
+        local arch
+        extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name "${package}"
+        printf 'WHERE `repository_stabilities`.`name`="testing"'
+        printf ' AND `binary_packages`.`%s`=from_base64("%s")' \
+          'pkgname'    "$(printf '%s' "${pkgname}" | base64 -w0)" \
+          'epoch'      "$(printf '%s' "${epoch}" | base64 -w0)" \
+          'pkgver'     "$(printf '%s' "${pkgver}" | base64 -w0)" \
+          'pkgrel'     "$(printf '%s' "${pkgrel}" | base64 -w0)" \
+          'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)"
+        printf ' AND `architectures`.`name`=from_base64("%s")' \
+          "$(printf '%s' "${arch}" | base64 -w0)"
+      )
+    ;;
+    'block'|'unblock')
+      # we expect a package source identifier (pkgbase.git_revision.mod_git_revision.repository)
+      # and we will only operate on packages in "unbuilt" repositories
+      # shellcheck disable=SC2016
+      combiner=$(
+        printf '`build_assignments`'
+        printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+          'package_sources'        'package_sources'        'build_assignments' 'package_source' \
+          'upstream_repositories'  'upstream_repositories'  'package_sources'   'upstream_package_repository' \
+          'binary_packages'        'build_assignments'      'binary_packages'   'build_assignment' \
+          'repositories'           'repositories'           'binary_packages'   'repository' \
+          'repository_stabilities' 'repository_stabilities' 'repositories'      'stability'
+      )
+      # shellcheck disable=SC2016
+      selector=$(
+        repository="${package##*.}"
+        pkgbase="${package%.*}"
+        mod_git_revision="${pkgbase##*.}"
+        pkgbase="${pkgbase%.*}"
+        git_revision="${pkgbase##*.}"
+        pkgbase="${pkgbase%.*}"
+        printf 'WHERE `repository_stabilities`.`name`="unbuilt"'
+        printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+          "$(printf '%s' "${repository}" | base64 -w0)"
+        printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+          'pkgbase'          "$(printf '%s' "${pkgbase}" | base64 -w0)" \
+          'git_revision'     "$(printf '%s' "${git_revision}" | base64 -w0)" \
+          'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)"
+      )
+    ;;
+    *)
+      >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
+      exit 42
+    ;;
+  esac
+  case "${action}" in
+    'block')
+      if [ -z "${reason}" ]; then
+        >&2 printf 'No reason is given for blocking package "%s".\n' "${package}"
+        exit 2
       fi
-    fi
-    case "${action}" in
-      'block')
-        if [ -z "${reason}" ]; then
-          >&2 printf 'No reason is given for blocking package "%s".\n' "${package}"
-          err=2
-        else
-          echo "${reason}" > \
-            "${work_dir}/package-states/${package}.blocked"
-          # shellcheck disable=SC2016
-          {
-            printf 'UPDATE `build_assignments`'
-            printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
-            printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
-            printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \
-              "$(printf '%s' "${reason}" | base64 -w0)"
-            printf ' WHERE'
-            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
-              'pkgbase' "${pkgbase}" \
-              'git_revision' "${git_rev}" \
-              'mod_git_revision' "${mod_git_rev}"
-            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
-              "${repo}"
-          } | tee /dev/stderr | \
-            ${mysql_command}
-          printf '%s %s\n' "${package}" "${reason}"
-        fi
-      ;;
-      'faulty')
-        if [ -f "${work_dir}/package-states/${package}.tested" ]; then
-          mv \
-            "${work_dir}/package-states/${package}.tested" \
-            "${work_dir}/package-states/${package}.testing"
-          # shellcheck disable=SC2016
-          {
-            printf 'UPDATE `binary_packages`'
-            printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-              'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
-              'package_sources' 'package_sources' 'build_assignments' 'package_source' \
-              'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
-            printf ' SET `binary_packages`.`has_issues`=1, `binary_packages`.`is_tested`=0'
-            printf ' WHERE'
-            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
-              'pkgbase' "${pkgbase}" \
-              'git_revision' "${git_rev}" \
-              'mod_git_revision' "${mod_git_rev}"
-            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
-              "${repo}"
-          } | \
-            ${mysql_command}
-          printf '%s\n' "${package}"
-        fi
-      ;;
-      'tested')
-        if [ -f "${work_dir}/package-states/${package}.testing" ] && \
-          ! grep -qF "[${package%.*.*.*}]" "${tmp_dir}/package-bug-titles"; then
-          mv \
-            "${work_dir}/package-states/${package}.testing" \
-            "${work_dir}/package-states/${package}.tested"
-          # shellcheck disable=SC2016
-          {
-            printf 'UPDATE `binary_packages`'
-            printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-              'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
-              'package_sources' 'package_sources' 'build_assignments' 'package_source' \
-              'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
-            printf ' SET `binary_packages`.`is_tested`=1'
-            printf ' WHERE'
-            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
-              'pkgbase' "${pkgbase}" \
-              'git_revision' "${git_rev}" \
-              'mod_git_revision' "${mod_git_rev}"
-            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
-              "${repo}"
-          } | \
-            ${mysql_command}
-          printf '%s\n' "${package}"
-        fi
-      ;;
-      'unblock')
-        if [ -f "${work_dir}/package-states/${package}.blocked" ]; then
-          rm "${work_dir}/package-states/${package}.blocked"
-          # shellcheck disable=SC2016
-          {
-            printf 'UPDATE `build_assignments`'
-            printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
-            printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
-            printf ' SET `build_assignments`.`is_blocked`=NULL'
-            printf ' WHERE'
-            printf ' `package_sources`.`%s`=from_base64("%s") AND' \
-              'pkgbase' "${pkgbase}" \
-              'git_revision' "${git_rev}" \
-              'mod_git_revision' "${mod_git_rev}"
-            printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
-              "${repo}"
-          } | \
-            ${mysql_command}
-          printf '%s\n' "${package}"
-        fi
-      ;;
-      *)
-        >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
-        exit 42
-      ;;
-    esac
-  done > \
-    "${output_file}"
-
-  exit ${err}
-} < \
-  "${input_file}"
+      tester='1'
+      # shellcheck disable=SC2016
+      modifier=$(
+        printf '`build_assignments`.`is_blocked`=from_base64("%s")' \
+          "$(printf '%s' "${reason}" | base64 -w0)"
+      )
+    ;;
+    'unblock')
+      # shellcheck disable=SC2016
+      tester='NOT `build_assignments`.`is_blocked` IS NULL'
+      # shellcheck disable=SC2016
+      modifier='`build_assignments`.`is_blocked`=NULL'
+    ;;
+    'faulty')
+      # shellcheck disable=SC2016
+      tester='`binary_packages`.`has_issues`=0'
+      # shellcheck disable=SC2016
+      modifier='`binary_packages`.`is_tested`=0,`binary_packages`.`has_issues`=1'
+    ;;
+    'tested')
+      # shellcheck disable=SC2016
+      tester='`binary_packages`.`is_tested`=0'
+      # shellcheck disable=SC2016
+      modifier='`binary_packages`.`is_tested`=1'
+    ;;
+    *)
+      >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
+      exit 42
+    ;;
+  esac
+  if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \
+    ${mysql_command} --raw --batch | \
+    sed '1d' | \
+    grep -qxF '1'; then
+    printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" | \
+      ${mysql_command}
+  fi
+done \
+  < "${input_file}" \
+  > "${output_file}"
-- 
cgit v1.2.3-54-g00ecf


From fd6b5522d364476a494f97ebf6248303b999f60d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 16:03:52 +0100
Subject: bin/modify-package-state: remove solved TODO

---
 bin/modify-package-state | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 477f3c3..aa42764 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -23,8 +23,6 @@ usage() {
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
-# TODO: binary packages should not be identified by revisions, but by versions
-
 eval set -- "$(
   getopt -o bfhntu \
     --long block \
-- 
cgit v1.2.3-54-g00ecf


From 65d82cf9a76c8442248a9eaf77a053c47c2ae415 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 20:26:39 +0100
Subject: bin/modify-package-state: bugfix - how could this have slipped
 through shellcheck?

---
 bin/modify-package-state | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index aa42764..5e4face 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -144,14 +144,8 @@ while read -r package reason; do
           'repository_stabilities' 'repository_stabilities' 'repositories'    'stability' \
           'architectures'          'architectures'          'binary_packages' 'architecture'
       )
-      # shellcheck disable=SC2016
+      # shellcheck disable=SC2016,SC2031
       selector=$(
-        local pkgname
-        local epoch
-        local pkgver
-        local pkgrel
-        local sub_pkgrel
-        local arch
         extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name "${package}"
         printf 'WHERE `repository_stabilities`.`name`="testing"'
         printf ' AND `binary_packages`.`%s`=from_base64("%s")' \
-- 
cgit v1.2.3-54-g00ecf


From 9baf70d2df76a5386208daecc6ce8fec11c1d560 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 21:17:08 +0100
Subject: bin/modify-package-state: we should modify the files, too - for now

---
 bin/modify-package-state | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 5e4face..3ae5c58 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -235,6 +235,33 @@ while read -r package reason; do
     grep -qxF '1'; then
     printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" | \
       ${mysql_command}
+    case "${action}" in
+      'block')
+        printf '%s' "${reason}" > \
+          "${work_dir}/package-states/${package}.blocked"
+      ;;
+      'unblock')
+        rm -f "${work_dir}/package-states/${package}.blocked"
+      ;;
+      'faulty')
+        if [ -f "${work_dir}/package-states/${package}.tested" ]; then
+          mv \
+            "${work_dir}/package-states/${package}.tested" \
+            "${work_dir}/package-states/${package}.testing"
+        fi
+      ;;
+      'tested')
+        if [ -f "${work_dir}/package-states/${package}.testing" ]; then
+          mv \
+            "${work_dir}/package-states/${package}.testing" \
+            "${work_dir}/package-states/${package}.tested"
+        fi
+      ;;
+      *)
+        >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
+        exit 42
+      ;;
+    esac
   fi
 done \
   < "${input_file}" \
-- 
cgit v1.2.3-54-g00ecf


From 12f0a85eccf4803af088399843fe915a598f3f20 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 31 Jan 2018 22:19:57 +0100
Subject: bin/build-master-status-from-mysql: check tested packages

---
 bin/build-master-status-from-mysql | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index b1bbb98..44e07c1 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -72,6 +72,42 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
   sort > \
   "${tmp_dir}/loops.file"
 
+# shellcheck disable=SC2016
+{
+  printf 'SELECT '
+  printf '`binary_packages`.`%s`,' \
+    'pkgname' \
+    'epoch' \
+    'pkgver' \
+    'pkgrel' \
+    'sub_pkgrel'
+  printf '`architectures`.`name`'
+  printf ' FROM `binary_packages`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'repositories'           'repositories'           'binary_packages' 'repository' \
+    'architectures'          'architectures'          'binary_packages' 'architecture' \
+    'repository_stabilities' 'repository_stabilities' 'repositories'    'stability'
+  printf ' WHERE `binary_packages`.`is_tested`'
+  printf ' AND NOT `binary_packages`.`has_issues`'
+  printf ' AND `repository_stabilities`.`name`="testing"'
+} | \
+  ${mysql_command} --batch | \
+  sed '
+    1d
+    s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
+  ' | \
+  sort -u > \
+  "${tmp_dir}/tested.mysql"
+find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
+  -name '*.tested' \
+  -exec cat '{}' \; | \
+  sed '
+    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0-\2/
+    s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
+  ' | \
+  sort -u > \
+  "${tmp_dir}/tested.file"
+
 {
   mysql_sanity_check || true
   diff -u \
@@ -86,6 +122,10 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
     "${tmp_dir}/loops.file" \
     "${tmp_dir}/loops.mysql" || \
     true
+  diff -u \
+    "${tmp_dir}/tested.file" \
+    "${tmp_dir}/tested.mysql" || \
+    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
-- 
cgit v1.2.3-54-g00ecf


From 0553c6d12a5e310899e8a5679913faafa72b17fb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 09:57:59 +0100
Subject: bin/build-master-status: write todos into database

---
 bin/build-master-status | 57 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/bin/build-master-status b/bin/build-master-status
index 4d47976..2477e3b 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -502,6 +502,7 @@ if ${web}; then
         s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
         p
       ' | \
+      tee "${tmp_dir}/todos" | \
       sed '
         :a
         N
@@ -517,6 +518,62 @@ if ${web}; then
   } > \
     "${tmp_dir}/todos.html"
 
+  if [ -s "${tmp_dir}/todos" ]; then
+    sed '
+      :a
+      N
+      /\n$/!ba
+      s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 |
+      s/\n$//
+      s/\n/\\n/g
+    ' -i "${tmp_dir}/todos"
+    while read -r file line desc; do
+      printf '%s %s %s\n' \
+        "$(printf '%s' "${file}" | base64 -w0)" \
+        "$(printf '%s' "${line}" | base64 -w0)" \
+        "$(printf '%s' "${desc}" | base64 -w0)"
+    done < \
+      "${tmp_dir}/todos" | \
+      sponge "${tmp_dir}/todos"
+    # update todos
+    # shellcheck disable=SC2016
+    while read -r file line desc; do
+      printf 'UPDATE TABLE `todos`'
+      printf ' SET `todos`.`line`=from_base64("%s")' \
+        "${line}"
+      printf ' WHERE `todos`.`file`=from_base64("%s")' \
+        "${file}"
+      printf ' AND `todos`.`desc`=from_base64("%s");\n' \
+        "${desc}"
+
+      printf 'UPDATE TABLE `todos`'
+      printf ' SET `todos`.`desc`=from_base64("%s")' \
+        "${desc}"
+      printf ' WHERE `todos`.`file`=from_base64("%s")' \
+        "${file}"
+      printf ' AND `todos`.`line`=from_base64("%s");\n' \
+        "${line}"
+    done | \
+      ${mysql_command}
+    # insert unfound todos
+    # shellcheck disable=SC2016
+    while read -r file line desc; do
+      printf ' ('
+      printf 'from_base64("%s"),' \
+        "${file}" \
+        "${line}" \
+        "${desc}" | \
+        sed 's/,$/),/'
+    done < \
+      "${tmp_dir}/todos" | \
+      sed '
+        1 i INSERT IGNORE INTO `todos` (`file`,`line`,`description`) VALUES
+        s/,$/;\n/
+      ' | \
+      ${mysql_command}
+    rm -f "${tmp_dir}/todos"
+  fi
+
   {
     printf '%s\n' \
       '<html>' \
-- 
cgit v1.2.3-54-g00ecf


From b8a0ed85a347f35a6c34723f80b4a7355f577785 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 10:09:11 +0100
Subject: bin/build-master-status: bugfix

---
 bin/build-master-status | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 2477e3b..1672789 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -538,22 +538,23 @@ if ${web}; then
     # update todos
     # shellcheck disable=SC2016
     while read -r file line desc; do
-      printf 'UPDATE TABLE `todos`'
+      printf 'UPDATE `todos`'
       printf ' SET `todos`.`line`=from_base64("%s")' \
         "${line}"
       printf ' WHERE `todos`.`file`=from_base64("%s")' \
         "${file}"
-      printf ' AND `todos`.`desc`=from_base64("%s");\n' \
+      printf ' AND `todos`.`description`=from_base64("%s");\n' \
         "${desc}"
 
-      printf 'UPDATE TABLE `todos`'
-      printf ' SET `todos`.`desc`=from_base64("%s")' \
+      printf 'UPDATE `todos`'
+      printf ' SET `todos`.`description`=from_base64("%s")' \
         "${desc}"
       printf ' WHERE `todos`.`file`=from_base64("%s")' \
         "${file}"
       printf ' AND `todos`.`line`=from_base64("%s");\n' \
         "${line}"
-    done | \
+    done < \
+      "${tmp_dir}/todos" | \
       ${mysql_command}
     # insert unfound todos
     # shellcheck disable=SC2016
-- 
cgit v1.2.3-54-g00ecf


From 8d3d7cabdad3b873722064632968f3ce916b3e99 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 11:51:58 +0100
Subject: add many mysql-TODOs

---
 bin/build-master-status            | 2 ++
 bin/build-master-status-from-mysql | 2 ++
 bin/build-packages                 | 2 ++
 bin/calculate-dependent-packages   | 4 ++++
 bin/check-bugtracker               | 2 ++
 bin/cleanup                        | 2 ++
 bin/db-update                      | 6 ++----
 bin/get-package-updates            | 2 ++
 bin/interpret-mail                 | 2 ++
 bin/prioritize-build-list          | 2 ++
 bin/return-assignment              | 2 ++
 bin/sanity-check                   | 2 ++
 bin/seed-build-list                | 2 ++
 bin/show-dependencies              | 2 ++
 bin/why-dont-you                   | 2 ++
 15 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 1672789..b117e4e 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -5,6 +5,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: replace by build-master-status-from-mysql
+
 usage() {
   >&2 echo ''
   >&2 echo 'build-master-status: report about status of build master'
diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 44e07c1..16353dd 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -5,6 +5,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: add all (necessary) features from build-master-status
+
 tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
diff --git a/bin/build-packages b/bin/build-packages
index 81afa7f..94292c7 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -10,6 +10,8 @@
 # TODO: report back in regular intervals, so the build master can
 # abort if the package is outdated and/or someone else finished it.
 
+# TODO: build other "architectures", too (pentium4, i486)
+
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
diff --git a/bin/calculate-dependent-packages b/bin/calculate-dependent-packages
index bfb87a1..3208e61 100755
--- a/bin/calculate-dependent-packages
+++ b/bin/calculate-dependent-packages
@@ -3,6 +3,10 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: save information in database
+
+# TODO: read information from database
+
 if [ -s "${work_dir}/build-master-sanity" ]; then
   >&2 echo 'Build master is not sane.'
   exit
diff --git a/bin/check-bugtracker b/bin/check-bugtracker
index 46e78db..6b304e3 100755
--- a/bin/check-bugtracker
+++ b/bin/check-bugtracker
@@ -8,6 +8,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: read information from database
+
 tmp_dir=$(mktemp -d 'tmp.check-bugtracker.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
diff --git a/bin/cleanup b/bin/cleanup
index 78daece..15b6fc9 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -5,6 +5,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: clean database, too
+
 # we only clean if run interactive or if no one is logged in
 if ! tty -s && \
   [ -n "$(users)" ]; then
diff --git a/bin/db-update b/bin/db-update
index fe45f41..35fd223 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -20,11 +20,9 @@
 # track packages in stable, too
 # know _exactly_ in which repository the packages are
 
-# TODO: fully integrate database (in other scripts too) - e.g. rely on
-# info in db instead of state-files
+# TODO: save information in database
 
-# TODO: add possibility to manually clone a (binary) package (e.g.
-# for build-support) - this should also be possible via email
+# TODO: read information from database
 
 # shellcheck disable=SC2039
 # shellcheck source=conf/default.conf
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 55efee4..91fb6be 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -11,6 +11,8 @@
 
 # TODO: test_exclusion does not yet cooperate with the database
 
+# TODO: read information from database
+
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
diff --git a/bin/interpret-mail b/bin/interpret-mail
index d16c07f..03336c2 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -5,6 +5,8 @@
 
 # TODO: enable email interface to delete packages
 
+# TODO: read information from database (?)
+
 if [ $# -ne 0 ]; then
   >&2 echo ''
   >&2 echo 'usage: interpret-mail'
diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list
index 06d4c06..5655223 100755
--- a/bin/prioritize-build-list
+++ b/bin/prioritize-build-list
@@ -3,6 +3,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: save information in database
+
 if [ $# -ne 1 ]; then
   echo 'Usage: prioritize-build-list pkg-regex'
   echo ' moves packages matching pkg-regex to front of build list'
diff --git a/bin/return-assignment b/bin/return-assignment
index 9de8add..42393ac 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -16,6 +16,8 @@
 
 # TODO: sign database
 
+# TODO: read information from database
+
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
diff --git a/bin/sanity-check b/bin/sanity-check
index 8736d47..e31a5ba 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -5,6 +5,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: read information from database
+
 usage() {
   >&2 echo ''
   >&2 echo 'sanity-check [options] [checks]: check sanity of build master'
diff --git a/bin/seed-build-list b/bin/seed-build-list
index cd3152d..08f6a42 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -5,6 +5,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: read information from database
+
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
diff --git a/bin/show-dependencies b/bin/show-dependencies
index efdf745..ee7db19 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -3,6 +3,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: read information from database
+
 # Create a lock file.
 
 if [ $# -eq 0 ]; then
diff --git a/bin/why-dont-you b/bin/why-dont-you
index 2e3ea86..830048b 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -7,6 +7,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: read information from database
+
 action="$1"
 shift
 
-- 
cgit v1.2.3-54-g00ecf


From 296fc3012f41fd1380dec4a955b287dbb305f7ea Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 11:54:49 +0100
Subject: web-scripts/todos.php new

---
 web-scripts/todos.php | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 web-scripts/todos.php

diff --git a/web-scripts/todos.php b/web-scripts/todos.php
new file mode 100644
index 0000000..6c3b555
--- /dev/null
+++ b/web-scripts/todos.php
@@ -0,0 +1,62 @@
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`todos`.`id`," .
+  "`todos`.`file`," .
+  "`todos`.`line`," .
+  "`todos`.`description` " .
+  "FROM `todos`;"
+);
+
+if ($result -> num_rows > 0) {
+
+  while ($row = $result->fetch_assoc())
+    $knot_rows[$row["id"]] =
+      $row["file"]. " (line ".$row["line"]."):\\n".$row["description"];
+
+  unset($knots);
+  foreach ($knot_rows as $knot)
+    $knots=$knots . "\"" . $knot . "\";\n";
+
+}
+
+$result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`todo_links`.`dependent`," .
+  "`todo_links`.`depending_on` " .
+  "FROM `todo_links`;"
+);
+
+if ($result -> num_rows > 0) {
+  $count = 0;
+  while ($row = $result->fetch_assoc()) {
+    $link_rows[$count]["dependent"] =
+      $knot_rows[$row["dependent"]];
+    $link_rows[$count]["depending_on"] =
+      $knot_rows[$row["depending_on"]];
+    $count++;
+  }
+
+  unset($edges);
+  foreach ($link_rows as $link)
+    $edges=$edges . "\"" . $link["depending_on"] . "\" -> \"" . $link["dependent"] . "\";\n";
+}
+
+header ("Content-type: image/png");
+passthru(
+  "dot -Tpng -o/dev/stdout /dev/stdin <<EOF\n".
+  "digraph dependencies {\n" .
+  "fontname=dejavu;\n" .
+  $knots .
+  $edges .
+  "}\n" .
+  "EOF\n"
+);
+
+?>
-- 
cgit v1.2.3-54-g00ecf


From faab202ee4659899c4f49753afcac87b0042feee Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 12:13:13 +0100
Subject: web-scripts/todos.php: layout

---
 web-scripts/todos.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/web-scripts/todos.php b/web-scripts/todos.php
index 6c3b555..123fa35 100644
--- a/web-scripts/todos.php
+++ b/web-scripts/todos.php
@@ -52,6 +52,7 @@ header ("Content-type: image/png");
 passthru(
   "dot -Tpng -o/dev/stdout /dev/stdin <<EOF\n".
   "digraph dependencies {\n" .
+  "rankdir=LR;\n" .
   "fontname=dejavu;\n" .
   $knots .
   $edges .
-- 
cgit v1.2.3-54-g00ecf


From 804f05c76120d3e12dcf11858ec4f70e59a3d194 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 12:16:43 +0100
Subject: bin/build-packages: fix comment to not break php script

---
 bin/build-packages | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index 94292c7..7f009c3 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -10,7 +10,7 @@
 # TODO: report back in regular intervals, so the build master can
 # abort if the package is outdated and/or someone else finished it.
 
-# TODO: build other "architectures", too (pentium4, i486)
+# TODO: build other 'architectures', too (pentium4, i486)
 
 # shellcheck disable=SC2016
 usage() {
-- 
cgit v1.2.3-54-g00ecf


From 8841e43ff0fc604092306a6ef8f9bf6eaf1b4637 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 12:26:09 +0100
Subject: bin/common-functions: add some meta-TODOs

---
 bin/common-functions | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bin/common-functions b/bin/common-functions
index 03a177b..a845d24 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -2,9 +2,13 @@
 
 # contains functions used by more than one script
 
+# shellcheck disable=SC2039
+
 # TODO: include link depenendencies in run-depends metadata
 
-# shellcheck disable=SC2039
+# TODO: have full information (currently in files) in database
+
+# TODO: remove state files / metadata files
 
 if [ -z "${base_dir}" ]; then
   # just to make shellcheck happy
-- 
cgit v1.2.3-54-g00ecf


From c7ffdfcaea68b6815b4023c97ee234f8908d8f62 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 13:32:24 +0100
Subject: bin/check-bugtracker: use database and do not(!) change files anymore

---
 bin/check-bugtracker | 82 +++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 56 insertions(+), 26 deletions(-)

diff --git a/bin/check-bugtracker b/bin/check-bugtracker
index 6b304e3..72ab133 100755
--- a/bin/check-bugtracker
+++ b/bin/check-bugtracker
@@ -1,35 +1,65 @@
 #!/bin/sh
 
 # check the bug tracker for packages in testing and community-testing
-# with issues and mark these packages as "testing" if they are currently
-# marked as "tested"
+# with issues and mark these packages as "has_issues" if they are faulty
+# and vice versa
 
 # shellcheck disable=SC2039
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: read information from database
+bug_list=$(
+  curl -LSs 'https://bugs.archlinux32.org/index.php?export_list=Export%20Tasklist' | \
+    sed -n '
+      1d
+      s/^[^,]\+,"//
+      T
+      s/^\([^"]\+\)"\(,[^,]\+\)\{2\},"\([^"]\+\)".*$/"\1" "\3"/
+      T
+      p
+    '
+)
 
-tmp_dir=$(mktemp -d 'tmp.check-bugtracker.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-receive_buglist 'Testing' | \
-  tr ' ,;' '\n' | \
-  sed -n '
-    s/^\[//
-    T
-    s/]$//
-    T
-    p
-  ' | \
-  sort -u > \
-  "${tmp_dir}/faulty-packages"
-
-find "${work_dir}/package-states" -name '*.tested' -printf '%f\n' | \
-  sed '
-    s|\.[^.]\+$||
-    s|^\(.*\)\(\.[^.]\+\)\{3\}$|\1 \0|
-  ' | \
-  sort -k1,1 | \
-  join -1 1 -2 1 -o 1.2 - "${tmp_dir}/faulty-packages" | \
-  "${base_dir}/bin/modify-package-state" -n --faulty /dev/stdin
+# shellcheck disable=SC2016
+{
+  printf 'SELECT `repository_stabilities`.`id`,`repository_stabilities`.`bugtracker_category`'
+  printf ' FROM `repository_stabilities`'
+  printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL'
+} | \
+  ${mysql_command} --raw --batch | \
+  sed '1d' | \
+  while read -r stability_id category; do
+    for has_issues in '1:' '0:NOT '; do
+      printf 'UPDATE `binary_packages`'
+      printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`'
+      printf ' SET `has_issues`=%s' \
+        "${has_issues%:*}"
+      printf ' WHERE `repositories`.`stability`=%s' \
+        "${stability_id}"
+      printf ' AND `binary_packages`.`pkgname` %sIN (' \
+        "${has_issues#*:}"
+      printf '%s\n' "${bug_list}" | \
+        sed -n '
+          s/^"'"$(str_to_regex "${category}")"'" //
+          T
+          :a
+            /\[.*]/ {
+              s/^[^[]*\[//
+              T
+              h
+              s/].*$//
+              p
+              x
+              ba
+            }
+        ' | \
+        base64_encode_each | \
+        sed '
+          s/^/from_base64("/
+          s/$/")/
+        ' | \
+        tr '\n' ','
+      printf '"");\n'
+    done
+  done | tee /dev/stderr | \
+  ${mysql_command}
-- 
cgit v1.2.3-54-g00ecf


From e4fafecf4a2d73c995f5e5befe001f5a2b4a68fa Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 13:34:08 +0100
Subject: bin/common-functions: remove obsolete function

---
 bin/common-functions | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/bin/common-functions b/bin/common-functions
index a845d24..d7d63f0 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -1234,23 +1234,6 @@ print_list_of_archaic_packages() {
     sort -u
 }
 
-# receive_buglist $category
-# receive the buglist from bugs.archlinux32.org for $category (e.g. 'Testing')
-# and print it to stdout
-
-receive_buglist() {
-  local category
-  category="$1"
-  curl -LSs 'https://bugs.archlinux32.org/index.php?export_list=Export%20Tasklist' | \
-    sed '
-      1 d
-      /^[^,]\+,"Packages: '"${category}"'",/ ! d
-      s/^\([^,]\+,\)\{3\}//
-      s/^\("[^"]\+"\|[^"][^,]*\),"//
-      s/".*$//
-    '
-}
-
 # next_sub_pkgrel $package $git_revision $mod_git_revision $repository
 # giv out the next sub-pkgrel of the given package
 
-- 
cgit v1.2.3-54-g00ecf


From 9923c55b4db85f2777053f27f6873d2bf22142d0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 13:35:10 +0100
Subject: bin/modify-package-state: remove done TODO

---
 bin/modify-package-state | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 3ae5c58..dec3013 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -128,9 +128,6 @@ if ! flock -s -n 9; then
   exit
 fi
 
-# TODO: handle bug reports, too
-# receive_buglist 'Testing'
-
 while read -r package reason; do
   case "${action}" in
     'faulty'|'tested')
-- 
cgit v1.2.3-54-g00ecf


From 4d31b38cf29577615aef7404bdf9cdbfa9d6cf50 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 13:37:44 +0100
Subject: bin/db-update: remove obsolete TODO

---
 bin/db-update | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 35fd223..9662ec4 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -20,8 +20,6 @@
 # track packages in stable, too
 # know _exactly_ in which repository the packages are
 
-# TODO: save information in database
-
 # TODO: read information from database
 
 # shellcheck disable=SC2039
-- 
cgit v1.2.3-54-g00ecf


From a154303d59fc2d6e8928df6351dcf76e8aefe64b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 14:02:05 +0100
Subject: bin/build-master-status: update todos correctly

---
 bin/build-master-status | 59 ++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 44 insertions(+), 15 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index b117e4e..ce49332 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -540,7 +540,7 @@ if ${web}; then
     # update todos
     # shellcheck disable=SC2016
     while read -r file line desc; do
-      printf 'UPDATE `todos`'
+      printf 'UPDATE IGNORE `todos`'
       printf ' SET `todos`.`line`=from_base64("%s")' \
         "${line}"
       printf ' WHERE `todos`.`file`=from_base64("%s")' \
@@ -548,7 +548,7 @@ if ${web}; then
       printf ' AND `todos`.`description`=from_base64("%s");\n' \
         "${desc}"
 
-      printf 'UPDATE `todos`'
+      printf 'UPDATE IGNORE `todos`'
       printf ' SET `todos`.`description`=from_base64("%s")' \
         "${desc}"
       printf ' WHERE `todos`.`file`=from_base64("%s")' \
@@ -560,19 +560,48 @@ if ${web}; then
       ${mysql_command}
     # insert unfound todos
     # shellcheck disable=SC2016
-    while read -r file line desc; do
-      printf ' ('
-      printf 'from_base64("%s"),' \
-        "${file}" \
-        "${line}" \
-        "${desc}" | \
-        sed 's/,$/),/'
-    done < \
-      "${tmp_dir}/todos" | \
-      sed '
-        1 i INSERT IGNORE INTO `todos` (`file`,`line`,`description`) VALUES
-        s/,$/;\n/
-      ' | \
+    {
+      printf 'CREATE TEMPORARY TABLE `td` ('
+      printf '`file` VARCHAR(64), '
+      printf '`line` MEDIUMINT, '
+      printf '`description` VARCHAR(128), '
+      printf 'UNIQUE KEY `position` (`file`,`line`), '
+      printf 'UNIQUE KEY `content` (`file`,`description`)'
+      printf ');'
+      printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES '
+      while read -r file line desc; do
+        printf '('
+        printf 'from_base64("%s"),' \
+          "${file}" \
+          "${line}" \
+          "${desc}" | \
+          sed 's/,$/),/'
+      done < \
+        "${tmp_dir}/todos" | \
+        sed '
+          s/,$//
+        '
+      printf ';\n'
+      printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) '
+      printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` '
+      printf 'FROM `td`;\n'
+      printf 'DELETE FROM `todos` WHERE NOT EXISTS ('
+      printf 'SELECT * FROM `td`'
+      printf ' AND `td`.`%s`=`todos`.`%s`' \
+        'file' 'file' \
+        'line' 'line' \
+        'description' 'description' | \
+        sed 's/^ AND / WHERE /'
+      printf ');'
+      printf 'DROP TABLE `td`;\n'
+      printf 'DELETE FROM `todo_links` WHERE NOT EXISTS ('
+      printf 'SELECT * FROM `todos` '
+      printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`'
+      printf ') OR NOT EXISTS ('
+      printf 'SELECT * FROM `todos` '
+      printf 'WHERE `todos`.`id`=`todo_links`.`dependent`'
+      printf ');\n'
+    } | \
       ${mysql_command}
     rm -f "${tmp_dir}/todos"
   fi
-- 
cgit v1.2.3-54-g00ecf


From eec7bff763e119a05b3b16c2c1e7477eb381f86a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 14:12:20 +0100
Subject: bin/bootstrap-mysql,bin/mysql-functions: new column `priority` in
 `build_assignments`

---
 bin/bootstrap-mysql | 1 +
 bin/mysql-functions | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 193b0eb..ae51e25 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -82,6 +82,7 @@ if [ ! "$1" = 'slim' ]; then
       ' architecture SMALLINT :architectures' \
       ' is_blocked VARCHAR(128) NULL' \
       ' is_broken BIT' \
+      ' priority SMALLINT' \
       ' UNIQUE content package_source architecture' \
       'build_dependency_loops BIGINT' \
       ' loop MEDIUMINT' \
diff --git a/bin/mysql-functions b/bin/mysql-functions
index f3dcc5b..a17f7a2 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -310,8 +310,8 @@ mysql_generate_package_metadata() {
       fi
     } | \
       while read -r arch; do
-        printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`)'
-        printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL'
+        printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
+        printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0'
         printf ' FROM `architectures` JOIN `package_sources`'
         printf ' WHERE `architectures`.`name` = from_base64("%s")' \
           "$(
-- 
cgit v1.2.3-54-g00ecf


From a03d072cc472de7d2707b97079335d17025bd475 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 14:14:05 +0100
Subject: bin/get-assignment: honor `priority` of `build_assignments`

---
 bin/get-assignment | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 48e8d53..0c7915d 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -144,6 +144,7 @@ next_building=$(
         printf '%s' "$1" | \
           base64 -w0
       )"
+    printf '`build_assignments`.`priority`,'
     printf '('
       printf 'SELECT COUNT(*)'
       printf ' FROM `failed_builds`'
@@ -173,14 +174,14 @@ next_building=$(
       printf ' OR NOT '
       mysql_query_has_pending_dependencies '`build_assignments`.`id`'
     printf ')'
-    printf ' ORDER BY `requested` DESC, `trials`, `part_of_loop`, `build_assignments`.`id`'
+    printf ' ORDER BY `requested` DESC, `priority` DESC, `trials`, `part_of_loop`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
   } | \
     ${mysql_command} --batch --raw | \
     sed '
       1d
       y/\t/ /
-      s/^\(\S\+ \)\{3\}//
+      s/^\(\S\+ \)\{4\}//
     '
 )
 if [ -n "${next_building}" ]; then
-- 
cgit v1.2.3-54-g00ecf


From 95c2ffe83ed58a13272df53acc5023d547be0c6d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 14:26:10 +0100
Subject: bin/get-assignment: reset priority when handing out a build
 assignment

---
 bin/get-assignment | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 0c7915d..7e8d25f 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -69,8 +69,20 @@ hand_out_assignment() {
     printf ' `upstream_repositories`.`name` = from_base64("%s")' \
       "$(printf '%s' "$4" | base64 -w0)"
     printf ')'
-    printf ' WHERE `build_slaves`.`name`=from_base64("%s");' \
+    printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
       "$(printf '%s' "${slave}" | base64 -w0)"
+
+    printf 'UPDATE `build_assignments`'
+    printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+    printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
+    printf ' SET `build_assignments`.`priority`=0'
+    printf ' WHERE'
+    printf ' `package_sources`.`%s` = from_base64("%s") AND' \
+      'pkgbase'          "$(printf '%s' "$1" | base64 -w0)" \
+      'git_revision'     "$(printf '%s' "$2" | base64 -w0)" \
+      'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+    printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \
+      "$(printf '%s' "$4" | base64 -w0)"
   } | \
     ${mysql_command}
 
-- 
cgit v1.2.3-54-g00ecf


From 94d523412cc6818c6e898f0213ec00695c404836 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 14:27:08 +0100
Subject: bin/return-assignment: reset priority when getting back a successful
 build assignment

---
 bin/return-assignment | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/return-assignment b/bin/return-assignment
index 42393ac..6c0ae16 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -434,6 +434,7 @@ fi
   printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
   printf ' SET'
   printf ' `build_assignments`.`is_broken`=0,'
+  printf ' `build_assignments`.`priority`=0,'
   printf ' `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s")),' \
     "$(
       printf '%s' "${destination}" | \
-- 
cgit v1.2.3-54-g00ecf


From abf2c85b1d4b87a04dbb8d0d707f84308838861f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 14:28:08 +0100
Subject: bin/prioritize-build-list: set priority in database

---
 bin/prioritize-build-list | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list
index 5655223..021e5c2 100755
--- a/bin/prioritize-build-list
+++ b/bin/prioritize-build-list
@@ -3,8 +3,6 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: save information in database
-
 if [ $# -ne 1 ]; then
   echo 'Usage: prioritize-build-list pkg-regex'
   echo ' moves packages matching pkg-regex to front of build list'
@@ -31,6 +29,19 @@ fi
 } | \
   sponge "${work_dir}/build-list"
 
+# shellcheck disable=SC2016
+{
+  printf 'UPDATE `build_assignments`'
+  printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+  printf ' SET `build_assignments`.`priority`=('
+    printf 'SELECT COALESCE(MAX(`others`.`priority`),0)+1'
+    printf ' FROM `build_assignments` AS `others`'
+  printf ')'
+  printf ' WHERE `package_sources`.`pkgbase` REGEXP from_base64("%s");\n' \
+    "$(printf '%s' "$1" | base64 -w0)"
+} | \
+  ${mysql_command}
+
 # Remove the lock file
 
 rm -f "${build_list_lock_file}"
-- 
cgit v1.2.3-54-g00ecf


From e46909f8589a0dadffafbe1f59997be3252afb5f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 14:51:49 +0100
Subject: bin/prioritize-build-list: only set priority of matchin packages ;-)

---
 bin/prioritize-build-list | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list
index 021e5c2..d611e79 100755
--- a/bin/prioritize-build-list
+++ b/bin/prioritize-build-list
@@ -32,13 +32,20 @@ fi
 # shellcheck disable=SC2016
 {
   printf 'UPDATE `build_assignments`'
-  printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+    'package_sources' 'build_assignments' 'package_source'   'package_sources' \
+    'binary_packages' 'binary_packages'   'build_assignment' 'build_assignments' \
+    'repositories'    'binary_packages'   'repository'       'repositories'
   printf ' SET `build_assignments`.`priority`=('
-    printf 'SELECT COALESCE(MAX(`others`.`priority`),0)+1'
-    printf ' FROM `build_assignments` AS `others`'
+    printf 'SELECT COALESCE(MAX(`all_priorities`.`priority`),0)+1'
+    printf ' FROM ('
+      printf 'SELECT `others`.`priority`'
+      printf ' FROM `build_assignments` AS `others`'
+    printf ') AS `all_priorities`'
   printf ')'
-  printf ' WHERE `package_sources`.`pkgbase` REGEXP from_base64("%s");\n' \
+  printf ' WHERE `package_sources`.`pkgbase` REGEXP from_base64("%s")' \
     "$(printf '%s' "$1" | base64 -w0)"
+  printf ' AND `repositories`.`name`="build-list";\n'
 } | \
   ${mysql_command}
 
-- 
cgit v1.2.3-54-g00ecf


From 8873150c610b1a5e7053cd1a462561b72cb6785e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 15:24:31 +0100
Subject: bin/build-master-status-from-mysql: check tesing packages, too

---
 bin/build-master-status-from-mysql | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 16353dd..8e828f7 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -110,6 +110,42 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   sort -u > \
   "${tmp_dir}/tested.file"
 
+# shellcheck disable=SC2016
+{
+  printf 'SELECT '
+  printf '`binary_packages`.`%s`,' \
+    'pkgname' \
+    'epoch' \
+    'pkgver' \
+    'pkgrel' \
+    'sub_pkgrel'
+  printf '`architectures`.`name`'
+  printf ' FROM `binary_packages`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'repositories'           'repositories'           'binary_packages' 'repository' \
+    'architectures'          'architectures'          'binary_packages' 'architecture' \
+    'repository_stabilities' 'repository_stabilities' 'repositories'    'stability'
+  printf ' WHERE NOT `binary_packages`.`is_tested`'
+  printf ' AND NOT `binary_packages`.`has_issues`'
+  printf ' AND `repository_stabilities`.`name`="testing"'
+} | \
+  ${mysql_command} --batch | \
+  sed '
+    1d
+    s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
+  ' | \
+  sort -u > \
+  "${tmp_dir}/testing.mysql"
+find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
+  -name '*.testing' \
+  -exec cat '{}' \; | \
+  sed '
+    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0-\2/
+    s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
+  ' | \
+  sort -u > \
+  "${tmp_dir}/testing.file"
+
 {
   mysql_sanity_check || true
   diff -u \
@@ -128,6 +164,10 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     "${tmp_dir}/tested.file" \
     "${tmp_dir}/tested.mysql" || \
     true
+  diff -u \
+    "${tmp_dir}/testing.file" \
+    "${tmp_dir}/testing.mysql" || \
+    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
-- 
cgit v1.2.3-54-g00ecf


From fbe2bc8a42570944d2b5fc8471bc43c568bdf3c2 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 16:18:10 +0100
Subject: bin/copy-to-build-support: copies packages on master mirror, to

---
 bin/copy-to-build-support | 87 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 77 insertions(+), 10 deletions(-)

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 85d73e5..f9eb53c 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -5,6 +5,9 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
 # shellcheck disable=SC2016
 {
   to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture'
@@ -20,7 +23,53 @@
     ' | \
     while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do
 
-      # TODO: move packages on master mirror, too!
+      id=$(
+        {
+          printf 'SELECT `binary_packages`.`id`,`repositories`.`name`'
+          printf ' FROM `binary_packages`'
+          printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+          printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+          printf ' WHERE'
+          printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
+            'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
+            'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
+            'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
+            'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \
+            'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)"
+          printf ' `architectures`.`name`=from_base64("%s")' \
+            "$(printf '%s' "${architecture}" | base64 -w0)"
+          printf ' LIMIT 1;\n'
+        } | \
+          ${mysql_command} --raw --batch | \
+          sed '
+            1d
+            y/\t/ /
+          '
+      )
+      if [ -z "${id}" ]; then
+        continue
+      fi
+      repository="${id#* }"
+      id="${id%% *}"
+
+      for suffix in '' '.sig'; do
+        printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \
+          "${repository}" \
+          "${package}" \
+          "${suffix}" \
+          "${package}" \
+          "${suffix}"
+      done >> \
+        "${tmp_dir}/sftp-command"
+      printf '%s/i686/%s/%s\n' \
+        "${master_mirror_rsync_directory}" \
+        "${repository}" \
+        "${package}" | \
+        sed '
+          p
+          s/$/.sig/
+        ' >> \
+        "${tmp_dir}/to-copy"
 
       printf 'INSERT IGNORE INTO `binary_packages`'
       printf ' (`repository`'
@@ -34,15 +83,33 @@
       printf ' FROM `binary_packages`'
       printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
       printf ' WHERE'
-      printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-        'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
-        'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
-        'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
-        'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \
-        'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)"
-      printf ' `architectures`.`name`=from_base64("%s")' \
-        "$(printf '%s' "${architecture}" | base64 -w0)"
-      printf ';\n'
+      printf ' `binary_packages`.`id`=%s;\n' \
+        "${id}"
     done
 } | \
   ${mysql_command}
+
+${master_mirror_rsync_command} \
+  "${master_mirror_rsync_directory}/i686/build-support/build-support.db."* \
+  "${master_mirror_rsync_directory}/i686/build-support/build-support.files."* \
+  "${tmp_dir}/"
+
+if [ -s "${tmp_dir}/to-copy" ]; then
+  mkdir "${tmp_dir}/transit/"
+  # shellcheck disable=SC2046
+  ${master_mirror_rsync_command} \
+    $(cat "${tmp_dir}/to-copy") \
+    "${tmp_dir}/transit/"
+  repo-add "${tmp_dir}/build-support.db.tar.gz" \
+    "${tmp_dir}/transit/"*".pkg.tar.xz"
+fi
+
+if [ -s "${tmp_dir}/sftp-command" ]; then
+  ${master_mirror_sftp_command} < \
+    "${tmp_dir}/sftp-command"
+fi
+
+${master_mirror_rsync_command} \
+  "${tmp_dir}/build-support.db."* \
+  "${tmp_dir}/build-support.files."* \
+  "${master_mirror_rsync_directory}/i686/build-support/"
-- 
cgit v1.2.3-54-g00ecf


From 75eee3c091714d13638afdfadc79b8d29537f22f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 16:35:05 +0100
Subject: bin/copy-to-build-support: removing of old build-support packages
 started

---
 bin/copy-to-build-support | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index f9eb53c..8a25f05 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -52,6 +52,8 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
       repository="${id#* }"
       id="${id%% *}"
 
+      printf '%s' "${package}" >> \
+        "${tmp_dir}/packages"
       for suffix in '' '.sig'; do
         printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \
           "${repository}" \
@@ -113,3 +115,11 @@ ${master_mirror_rsync_command} \
   "${tmp_dir}/build-support.db."* \
   "${tmp_dir}/build-support.files."* \
   "${master_mirror_rsync_directory}/i686/build-support/"
+
+# TODO: This does not work correctly yet!
+if [ -s "${tmp_dir}/packages" ]; then
+  while read -r package; do
+    remove_old_package_versions 'i686' 'build-support' "${package}"
+  done < \
+    "${tmp_dir}/packages"
+fi
-- 
cgit v1.2.3-54-g00ecf


From 3a9e15c190799046ee665cd015ba9371fcf09b4a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 09:41:34 +0100
Subject: bin/common-functions: add TODO

---
 bin/common-functions | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/common-functions b/bin/common-functions
index d7d63f0..85e7673 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -400,6 +400,9 @@ ls_master_mirror() {
 
 }
 
+# TODO: the actions of remove_old_package_versions should be done
+# on basis of the information in the database and
+
 # remove_old_package_versions $arch $repository $package_file
 
 # removes all older (not-newer) versions of $package_file
-- 
cgit v1.2.3-54-g00ecf


From ec0fadd3426af36f55ac8d7082e4b2dca957ae56 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 09:42:38 +0100
Subject: bin/copy-to-build-support: finshed, should work as defined

---
 bin/copy-to-build-support | 135 ++++++++++++++++++++++++----------------------
 1 file changed, 70 insertions(+), 65 deletions(-)

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 8a25f05..2fa43d1 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -8,71 +8,72 @@
 tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX")
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-# shellcheck disable=SC2016
-{
-  to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture'
-  printf '%s\n' "$@" | \
-    sed -n '
-      s/.\+/\0 \0/
-      T
-      s/\.pkg\.tar\.xz$//
-      s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/
-      s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/
-      s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/
-      p
-    ' | \
-    while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do
+to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture'
+printf '%s\n' "$@" | \
+  sed -n '
+    s/.\+/\0 \0/
+    T
+    s/\.pkg\.tar\.xz$//
+    s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/
+    s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/
+    s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/
+    p
+  ' | \
+  while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do
 
-      id=$(
-        {
-          printf 'SELECT `binary_packages`.`id`,`repositories`.`name`'
-          printf ' FROM `binary_packages`'
-          printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
-          printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
-          printf ' WHERE'
-          printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-            'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
-            'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
-            'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
-            'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \
-            'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)"
-          printf ' `architectures`.`name`=from_base64("%s")' \
-            "$(printf '%s' "${architecture}" | base64 -w0)"
-          printf ' LIMIT 1;\n'
-        } | \
-          ${mysql_command} --raw --batch | \
-          sed '
-            1d
-            y/\t/ /
-          '
-      )
-      if [ -z "${id}" ]; then
-        continue
-      fi
-      repository="${id#* }"
-      id="${id%% *}"
+    # shellcheck disable=SC2016
+    id=$(
+      {
+        printf 'SELECT `binary_packages`.`id`,`repositories`.`name`'
+        printf ' FROM `binary_packages`'
+        printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+        printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+        printf ' WHERE'
+        printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
+          'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
+          'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
+          'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
+          'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \
+          'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)"
+        printf ' `architectures`.`name`=from_base64("%s")' \
+          "$(printf '%s' "${architecture}" | base64 -w0)"
+        printf ' LIMIT 1;\n'
+      } | \
+        ${mysql_command} --raw --batch | \
+        sed '
+          1d
+          y/\t/ /
+        '
+    )
+    if [ -z "${id}" ]; then
+      continue
+    fi
+    repository="${id#* }"
+    id="${id%% *}"
 
-      printf '%s' "${package}" >> \
-        "${tmp_dir}/packages"
-      for suffix in '' '.sig'; do
-        printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \
-          "${repository}" \
-          "${package}" \
-          "${suffix}" \
-          "${package}" \
-          "${suffix}"
-      done >> \
-        "${tmp_dir}/sftp-command"
-      printf '%s/i686/%s/%s\n' \
-        "${master_mirror_rsync_directory}" \
+    printf '%s\n' "${package}" >> \
+      "${tmp_dir}/packages"
+    for suffix in '' '.sig'; do
+      printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \
         "${repository}" \
-        "${package}" | \
-        sed '
-          p
-          s/$/.sig/
-        ' >> \
-        "${tmp_dir}/to-copy"
+        "${package}" \
+        "${suffix}" \
+        "${package}" \
+        "${suffix}"
+    done >> \
+      "${tmp_dir}/sftp-command"
+    printf '%s/i686/%s/%s\n' \
+      "${master_mirror_rsync_directory}" \
+      "${repository}" \
+      "${package}" | \
+      sed '
+        p
+        s/$/.sig/
+      ' >> \
+      "${tmp_dir}/to-copy"
 
+    # shellcheck disable=SC2016
+    {
       printf 'INSERT IGNORE INTO `binary_packages`'
       printf ' (`repository`'
       # shellcheck disable=SC2086
@@ -87,9 +88,9 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
       printf ' WHERE'
       printf ' `binary_packages`.`id`=%s;\n' \
         "${id}"
-    done
-} | \
-  ${mysql_command}
+    } >> \
+      "${tmp_dir}/mysql-command"
+  done
 
 ${master_mirror_rsync_command} \
   "${master_mirror_rsync_directory}/i686/build-support/build-support.db."* \
@@ -116,7 +117,11 @@ ${master_mirror_rsync_command} \
   "${tmp_dir}/build-support.files."* \
   "${master_mirror_rsync_directory}/i686/build-support/"
 
-# TODO: This does not work correctly yet!
+if [ -s "${tmp_dir}/mysql-command" ]; then
+  ${mysql_command} < \
+    "${tmp_dir}/mysql-command"
+fi
+
 if [ -s "${tmp_dir}/packages" ]; then
   while read -r package; do
     remove_old_package_versions 'i686' 'build-support' "${package}"
-- 
cgit v1.2.3-54-g00ecf


From dc6742f7c7243e734d8ce278359e278674f10bd3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 09:45:07 +0100
Subject: bin/interpret-mail: reorder help alphabetically

---
 bin/interpret-mail | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/interpret-mail b/bin/interpret-mail
index 03336c2..08e0f16 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -23,15 +23,15 @@ if [ $# -ne 0 ]; then
   >&2 echo '  - "block: <state-file> <reason>":'
   >&2 echo '    Block the given packge for the given reason.'
   >&2 echo ''
+  >&2 echo '  - "schedule: <pkgbase>":'
+  >&2 echo '    Put the given package on the build list (again).'
+  >&2 echo ''
   >&2 echo '  - "stabilize: <package-file>":'
   >&2 echo '    Mark the given package as tested.'
   >&2 echo ''
   >&2 echo '  - "unblock: <state-file>":'
   >&2 echo '    Unblock the given packge.'
   >&2 echo ''
-  >&2 echo '  - "schedule: <pkgname>":'
-  >&2 echo '    Put the given package on the build list (again).'
-  >&2 echo ''
   >&2 echo '  - ALL: all of the above (only valid in'
   >&2 echo '    "conf/admin-gpg-keys")'
   >&2 echo ''
-- 
cgit v1.2.3-54-g00ecf


From fdf371775ebbc5b70fc0df755259b8a7acce41d3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 09:53:40 +0100
Subject: bin/copy-to-build-support: expect packages in file instead of
 arguments

---
 bin/copy-to-build-support | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 2fa43d1..4710012 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -5,20 +5,25 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+if [ "$#" -ne 1 ]; then
+  >&2 echo 'usage:'
+  >&2 echo '  copy-to-build-support package-list'
+  exit 2
+fi
+
 tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX")
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
 to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture'
-printf '%s\n' "$@" | \
-  sed -n '
-    s/.\+/\0 \0/
-    T
-    s/\.pkg\.tar\.xz$//
-    s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/
-    s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/
-    s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/
-    p
-  ' | \
+sed -n '
+  s/.\+/\0 \0/
+  T
+  s/\.pkg\.tar\.xz$//
+  s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/
+  s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/
+  s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/
+  p
+' "$1" | \
   while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do
 
     # shellcheck disable=SC2016
@@ -128,3 +133,8 @@ if [ -s "${tmp_dir}/packages" ]; then
   done < \
     "${tmp_dir}/packages"
 fi
+
+if [ -w "$1" ]; then
+  cat "${tmp_dir}/packages" > \
+    "$1"
+fi
-- 
cgit v1.2.3-54-g00ecf


From 96a254701d2c503bb052a8f2937f5b7c22ca01be Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 09:55:00 +0100
Subject: bin/interpret-mail: copy-to-build-support new

---
 bin/interpret-mail | 14 ++++++++++++++
 conf/default.conf  |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/bin/interpret-mail b/bin/interpret-mail
index 08e0f16..1cc466e 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -23,6 +23,9 @@ if [ $# -ne 0 ]; then
   >&2 echo '  - "block: <state-file> <reason>":'
   >&2 echo '    Block the given packge for the given reason.'
   >&2 echo ''
+  >&2 echo '  - "copy-to-build-support: <pkgname>":'
+  >&2 echo '    Copy the given binary package into [build-support].'
+  >&2 echo ''
   >&2 echo '  - "schedule: <pkgbase>":'
   >&2 echo '    Put the given package on the build list (again).'
   >&2 echo ''
@@ -226,3 +229,14 @@ if [ -s "${tmp_dir}/schedule" ]; then
     sponge "${tmp_dir}/schedule"
   log 'Successfully (re)scheduled %s packages.\n' "$(wc -l < "${tmp_dir}/schedule")"
 fi
+
+if [ -s "${tmp_dir}/copy-to-build-support" ]; then
+  sed -i '
+    /\.pkg\.tar\.xz$/!s/$/.pkg.tar.xz/
+  ' "${tmp_dir}/copy-to-build-support"
+  if run_and_log_on_error "${base_dir}/bin/copy-to-build-support" "${tmp_dir}/copy-to-build-support"; then
+    log 'Successfully copied %s packages to [build-support].\n' "$(wc -l < "${tmp_dir}/copy-to-build-support")"
+  else
+    log 'There was an error while copying the packages to [build-support] - ignoring this message.\n'
+  fi
+fi
diff --git a/conf/default.conf b/conf/default.conf
index 6ee991b..84c0e4f 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -35,7 +35,7 @@ repo_key='0xdeadbeef'
 package_key='0x15eebadc0de'
 
 # what can be done via the email interface
-possible_email_actions='stabilize block unblock schedule'
+possible_email_actions='stabilize block unblock schedule copy-to-build-support'
 
 # how many lines of email logs should be kept maximally
 max_mail_log_lines=10
-- 
cgit v1.2.3-54-g00ecf


From 79c2f647aab6c61d2c98b3a95bcafecbb87124fa Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 10:02:49 +0100
Subject: bin/common-functions: fix typo in comment

---
 bin/common-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/common-functions b/bin/common-functions
index 85e7673..3a03d27 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -401,7 +401,7 @@ ls_master_mirror() {
 }
 
 # TODO: the actions of remove_old_package_versions should be done
-# on basis of the information in the database and
+# on basis of the information in the database
 
 # remove_old_package_versions $arch $repository $package_file
 
-- 
cgit v1.2.3-54-g00ecf


From fb77df17de8de4535638b747956622b5640f78f8 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 10:29:15 +0100
Subject: bin/modify-package-state: binary packages are identified by version,
 thus we need to dig for the git revisions

---
 bin/modify-package-state | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index dec3013..a3b23f3 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -241,18 +241,22 @@ while read -r package reason; do
         rm -f "${work_dir}/package-states/${package}.blocked"
       ;;
       'faulty')
-        if [ -f "${work_dir}/package-states/${package}.tested" ]; then
-          mv \
-            "${work_dir}/package-states/${package}.tested" \
-            "${work_dir}/package-states/${package}.testing"
-        fi
+        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
+          -name '*.tested' \
+          -exec grep -lxF "${package}" '{}' \; | \
+          while read -r sf; do
+            mv "${sf}" "${sf%ed}ing"
+          done
       ;;
       'tested')
-        if [ -f "${work_dir}/package-states/${package}.testing" ]; then
-          mv \
-            "${work_dir}/package-states/${package}.testing" \
-            "${work_dir}/package-states/${package}.tested"
-        fi
+          echo "pkg: ${package}" >&2
+        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
+          -name '*.testing' \
+          -exec grep -lxF "${package}" '{}' \; | \
+          while read -r sf; do
+          echo "sf:  ${sf}" >&2
+            mv "${sf}" "${sf%ing}ed"
+          done
       ;;
       *)
         >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
-- 
cgit v1.2.3-54-g00ecf


From d240ec6c6329004f97eff2627d72de77c51664c0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 10:32:04 +0100
Subject: bin/mysql-functions: bugfix - I used the wrong variable

---
 bin/mysql-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index a17f7a2..d0861e7 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -39,7 +39,7 @@ mysql_add_package_source() {
   fi
   uses_upstream=0
   for repo in ${repo_names}; do
-    if eval 'git -C "${repo_paths__'"${values}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
+    if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
       tar -t 2>/dev/null | \
       sed 's,-x86_64/,-any/,' | \
       grep -qFx "$1/repos/$4-any/PKGBUILD"; then
-- 
cgit v1.2.3-54-g00ecf


From 879c73de4704e8531e0434316de09b184f5a43dc Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 10:44:52 +0100
Subject: bin/get-package-updates: update todo

---
 bin/get-package-updates | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 91fb6be..fc66ec6 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -86,7 +86,8 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
-# TODO: Should new deletion-list packages be recorded in the database, too?
+# TODO: New deletion-list packages should be recorded in the database,
+# too - including all binary information (especially dependencies!)
 
 # delete_package package
 # mark $package for deletion
-- 
cgit v1.2.3-54-g00ecf


From c2f8dbb0fa210ea32c59b5b87b3f2e900eb32d70 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 11:38:35 +0100
Subject: bin/seed-build-list: start mysql query

---
 bin/seed-build-list | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 08f6a42..9a0123e 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -133,6 +133,23 @@ mod_repo_rev=$(cat "${work_dir}/archlinux32.revision")
     generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}"
   done
 
+# shellcheck disable=SC2016
+{
+  printf 'SELECT '
+  printf '`package_sources`.`%s`,' \
+    'pkgbase' 'git_revision' 'mod_git_revision'
+  printf '`upstream_repositories`.`name`,`binary_packages`.`pkgname`'
+  printf ' FROM `binary_packages`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
+    'package_sources'       'package_sources'       'build_assignments' 'package_source' \
+    'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+} | \
+  ${mysql_command} --raw --batch | \
+  sed '
+    1d
+    y/\t/ /
+  ' > /dev/null # not yet working (the query works, but the entries are missing)
 find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
   sed '
     s|^.*/||
-- 
cgit v1.2.3-54-g00ecf


From 7906f6157963618a7b8f319f993eeffbf7bd2f40 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 11:42:09 +0100
Subject: bin/modify-package-state: remove debug output

---
 bin/modify-package-state | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index a3b23f3..79525cf 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -249,12 +249,10 @@ while read -r package reason; do
           done
       ;;
       'tested')
-          echo "pkg: ${package}" >&2
         find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
           -name '*.testing' \
           -exec grep -lxF "${package}" '{}' \; | \
           while read -r sf; do
-          echo "sf:  ${sf}" >&2
             mv "${sf}" "${sf%ing}ed"
           done
       ;;
-- 
cgit v1.2.3-54-g00ecf


From a8a0e3292863b7eb74b196e31db3cdae944c593a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 2 Feb 2018 16:19:20 +0100
Subject: web-scripts/build-list.php new

---
 web-scripts/build-list.php | 159 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 web-scripts/build-list.php

diff --git a/web-scripts/build-list.php b/web-scripts/build-list.php
new file mode 100644
index 0000000..1b5c359
--- /dev/null
+++ b/web-scripts/build-list.php
@@ -0,0 +1,159 @@
+<html>
+<head>
+<title>List of scheduled package builds</title>
+<link rel="stylesheet" type="text/css" href="/static/style.css">
+</head>
+<body>
+<a href="/build-logs/">build logs</a><br>
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`build_assignments`.`id`," .
+  "`build_assignments`.`is_blocked`," .
+  "`package_sources`.`pkgbase`," .
+  "`package_sources`.`git_revision`," .
+  "`package_sources`.`mod_git_revision`," .
+  "`package_sources`.`uses_upstream`," .
+  "`package_sources`.`uses_modification`," .
+  "`upstream_repositories`.`name` AS `package_repository`," .
+  "`git_repositories`.`name` AS `git_repository`," .
+  "`architectures`.`name` AS `arch`," .
+  "EXISTS (SELECT * " .
+    "FROM `binary_packages` `broken_bin` " .
+    "JOIN `dependencies` ON `dependencies`.`dependent` = `broken_bin`.`id` " .
+    "JOIN `install_target_providers` ON `install_target_providers`.`install_target` = `dependencies`.`depending_on` " .
+    "JOIN `binary_packages` `to_be_built` ON `to_be_built`.`id` = `install_target_providers`.`package` " .
+    "JOIN `repositories` ON `to_be_built`.`repository` = `repositories`.`id` " .
+    "WHERE `broken_bin`.`build_assignment`=`build_assignments`.`id` ".
+    "AND `repositories`.`name`=\"build-list\" " .
+    "AND `to_be_built`.`build_assignment`!=`build_assignments`.`id`" .
+  ") AS `dependencies_pending`," .
+  "(SELECT count(*) " .
+    "FROM `build_dependency_loops` " .
+    "WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`" .
+  ") AS `loops` " .
+  "FROM `build_assignments` " .
+  "JOIN `architectures` ON `build_assignments`.`architecture` = `architectures`.`id` " .
+  "JOIN `package_sources` ON `build_assignments`.`package_source` = `package_sources`.`id` " .
+  "JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id` " .
+  "JOIN `git_repositories` ON `upstream_repositories`.`git_repository`=`git_repositories`.`id` " .
+  "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
+  "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
+  "WHERE `repositories`.`name`=\"build-list\""
+);
+if ($result -> num_rows > 0) {
+
+  $count = 0;
+
+  while($row = $result->fetch_assoc()) {
+
+    $fail_result = $mysql -> query(
+      "SELECT " .
+      "`failed_builds`.`id` " .
+      "FROM `failed_builds` " .
+      "WHERE `failed_builds`.`build_assignment`=".$row["id"]
+    );
+
+    $rows[$count]["trials"] = $fail_result -> num_rows;
+
+    $rows[$count]["loops"] = $row["loops"];
+    $rows[$count]["pkgbase"] = $row["pkgbase"];
+    if ($row["dependencies_pending"]=="0")
+      $rows[$count]["pkgbase_print"] = $rows[$count]["pkgbase"];
+    else
+      $rows[$count]["pkgbase_print"] = "(" . $rows[$count]["pkgbase"] . ")";
+    if ($row["uses_upstream"]) {
+      $rows[$count]["git_revision"] =
+        "<a href=\"https://git.archlinux.org/svntogit/" .
+        $row["git_repository"] . ".git/tree/" .
+        $row["pkgbase"] . "/repos/" .
+        $row["package_repository"] . "-";
+      if ($row["arch"]=="any")
+        $rows[$count]["git_revision"] =
+          $rows[$count]["git_revision"] . "any";
+      else
+        $rows[$count]["git_revision"] =
+          $rows[$count]["git_revision"] . "x86_64";
+      $rows[$count]["git_revision"] =
+        $rows[$count]["git_revision"] . "/\">" .
+        $row["git_revision"] . "</a>";
+    } else
+      $rows[$count]["git_revision"] = $row["git_revision"];
+    if ($row["uses_modification"])
+      $rows[$count]["mod_git_revision"] =
+        "<a href=\"https://github.com/archlinux32/packages/tree/" .
+        $row["mod_git_revision"] . "/" .
+        $row["package_repository"] . "/" .
+        $row["pkgbase"] . "\">" .
+        $row["mod_git_revision"] . "</a>";
+    else
+      $rows[$count]["mod_git_revision"] = $row["mod_git_revision"];
+    $rows[$count]["package_repository"] = $row["package_repository"];
+    if ($row["is_blocked"]=="") {
+      $rows[$count]["is_blocked"]="&nbsp;";
+    }
+    else {
+      $rows[$count]["is_blocked"] = preg_replace(
+        array (
+          "/FS32#(\\d+)/",
+          "/FS#(\\d+)/"
+        ),
+        array (
+          "<a href=\"https://bugs.archlinux32.org/index.php?do=details&task_id=$1\">$0</a>",
+          "<a href=\"https://bugs.archlinux.org/task/$1\">$0</a>"
+        ),
+        $row["is_blocked"]
+      );
+    }
+    $count++;
+  }
+
+  usort(
+    $rows,
+    function (array $a, array $b) {
+      if ($a["trials"] < $b["trials"])
+        return -1;
+      if ($a["trials"] > $b["trials"])
+        return 1;
+      return strcmp($a["pkgbase"],$b["pkgbase"]);
+    }
+  );
+
+  print "<table>\n";
+  print "<tr>";
+  print "<th>package</th>";
+  print "<th>git revision</th>";
+  print "<th>modification git revision</th>";
+  print "<th>package repository</th>";
+  print "<th>compilations</th>";
+  print "<th>loops</th>";
+  print "<th>blocked</th>";
+  print "</tr>\n";
+
+  foreach($rows as $row) {
+
+    print "<tr>";
+
+    print "<td><a href=\"/graphs/".$row["pkgbase"].".png\">".$row["pkgbase_print"]."</a></td>";
+    print "<td><p style=\"font-size:8px\">".$row["git_revision"]."</p></td>";
+    print "<td><p style=\"font-size:8px\">".$row["mod_git_revision"]."</p></td>";
+    print "<td>".$row["package_repository"]."</td>";
+    print "<td>".$row["trials"]."</td>";
+    print "<td>".$row["loops"]."</td>";
+    print "<td>".$row["is_blocked"]."</td>";
+
+    print "</tr>\n";
+  }
+
+  print "</table>\n";
+}
+
+?>
+</body>
+</html>
-- 
cgit v1.2.3-54-g00ecf


From 2cece122e1fa57ed5cce7f29108798ee1004d261 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 3 Feb 2018 12:24:12 +0100
Subject: bin/get-assignment: add TODO

---
 bin/get-assignment | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/get-assignment b/bin/get-assignment
index 7e8d25f..85585c0 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -123,6 +123,8 @@ clean_up() {
 tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
 trap clean_up EXIT
 
+# TODO: only hand out package if it is still yet to be built
+
 # if we're building something already, hand it out (again)
 currently_building=$(
   # shellcheck disable=SC2016
-- 
cgit v1.2.3-54-g00ecf


From 37c1de524dad0a1363502ac2308f0091dd70b3d2 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 3 Feb 2018 13:39:50 +0100
Subject: bin/return-assignment: remove build assignment from other build
 slaves, too, if it was a successful one

---
 bin/return-assignment | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 6c0ae16..bf27232 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -447,9 +447,11 @@ fi
       printf '%s' "${slave}" | \
         base64 -w0
     )"
-  printf 'UPDATE `build_slaves` SET'
-  printf ' `build_slaves`.`currently_building`=NULL'
-  printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
+  printf 'UPDATE `build_slaves` AS `to_update`'
+  printf ' JOIN `build_slaves` AS `current_slave`'
+  printf ' ON `to_update`.`currently_building`=`current_slave`.`currently_building`'
+  printf ' SET `to_update`.`currently_building`=NULL'
+  printf ' WHERE `current_slave`.`name`=from_base64("%s");\n' \
     "$(
       printf '%s' "${slave}" | \
         base64 -w0
-- 
cgit v1.2.3-54-g00ecf


From 3ef5bfb56e453c9674f99e7f5a64b66835f58b8f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 3 Feb 2018 13:58:47 +0100
Subject: bin/get-assignment: only hand out the same package again if it is
 _still_ on the build-list

---
 bin/get-assignment | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 85585c0..67c7aa4 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -123,18 +123,19 @@ clean_up() {
 tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
 trap clean_up EXIT
 
-# TODO: only hand out package if it is still yet to be built
-
 # if we're building something already, hand it out (again)
 currently_building=$(
   # shellcheck disable=SC2016
   {
     printf 'SELECT '
     mysql_query_select_pkgbase_and_revision
-    printf ' JOIN `build_slaves`'
-    printf ' ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
+    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+      'build_slaves'    'build_slaves'    'currently_building' 'build_assignments' \
+      'binary_packages' 'binary_packages' 'build_assignment'   'build_assignments' \
+      'repositories'    'binary_packages' 'repository'         'repositories'
     printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
       "$(printf '%s' "${slave}" | base64 -w0)"
+    printf ' AND `repositories`.`name`="build-list"'
     printf ' LIMIT 1;\n'
   } | \
     ${mysql_command} --batch --raw | \
-- 
cgit v1.2.3-54-g00ecf


From c3279084e8f93a9f2f0488c3f78b72b8df3c65e4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 4 Feb 2018 08:36:53 +0100
Subject: bin/interpret-mail: add TODO

---
 bin/interpret-mail | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/interpret-mail b/bin/interpret-mail
index 1cc466e..b4a39d7 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -7,6 +7,8 @@
 
 # TODO: read information from database (?)
 
+# TODO: did/does not correctly report about the number of blocked packages
+
 if [ $# -ne 0 ]; then
   >&2 echo ''
   >&2 echo 'usage: interpret-mail'
-- 
cgit v1.2.3-54-g00ecf


From 27106d9e0c70524a42cad4b2d662177c8490c174 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 4 Feb 2018 12:07:03 +0100
Subject: bin/build-master-status: only add not-yet-existent todos newly to
 database

---
 bin/build-master-status | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index ce49332..0e80bd6 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -584,7 +584,16 @@ if ${web}; then
       printf ';\n'
       printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) '
       printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` '
-      printf 'FROM `td`;\n'
+      printf 'FROM `td` '
+      printf 'WHERE NOT EXISTS ('
+      printf 'SELECT * FROM `todos`'
+      printf ' AND `td`.`%s`=`todos`.`%s`' \
+        'file' 'file' \
+        'line' 'line' \
+        'description' 'description' | \
+        sed 's/^ AND / WHERE /'
+      printf ');\n'
+
       printf 'DELETE FROM `todos` WHERE NOT EXISTS ('
       printf 'SELECT * FROM `td`'
       printf ' AND `td`.`%s`=`todos`.`%s`' \
-- 
cgit v1.2.3-54-g00ecf


From 9ef68dd176d10bebf92cd937f8b6115a9d40b0bf Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 4 Feb 2018 13:40:19 +0100
Subject: bin/build-master-status: be more flexible when creating the temporary
 todos-table

---
 bin/build-master-status | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 0e80bd6..d74ff8a 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -561,13 +561,13 @@ if ${web}; then
     # insert unfound todos
     # shellcheck disable=SC2016
     {
-      printf 'CREATE TEMPORARY TABLE `td` ('
-      printf '`file` VARCHAR(64), '
-      printf '`line` MEDIUMINT, '
-      printf '`description` VARCHAR(128), '
-      printf 'UNIQUE KEY `position` (`file`,`line`), '
-      printf 'UNIQUE KEY `content` (`file`,`description`)'
-      printf ');'
+      printf 'SHOW CREATE TABLE `todos`' | \
+        ${mysql_command} --raw --batch | \
+        sed '
+          1d
+          2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
+        '
+      printf ';\n'
       printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES '
       while read -r file line desc; do
         printf '('
-- 
cgit v1.2.3-54-g00ecf


From 95f58cb2eb6bbb7143fc292458dde313c12a54be Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 4 Feb 2018 13:49:28 +0100
Subject: bin/modify-package-state: report, what has been done (so
 bin/interpret-mail reports correctly, too)

---
 bin/interpret-mail       | 2 --
 bin/modify-package-state | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/bin/interpret-mail b/bin/interpret-mail
index b4a39d7..1cc466e 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -7,8 +7,6 @@
 
 # TODO: read information from database (?)
 
-# TODO: did/does not correctly report about the number of blocked packages
-
 if [ $# -ne 0 ]; then
   >&2 echo ''
   >&2 echo 'usage: interpret-mail'
diff --git a/bin/modify-package-state b/bin/modify-package-state
index 79525cf..0703238 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -261,6 +261,7 @@ while read -r package reason; do
         exit 42
       ;;
     esac
+    printf '%s %s\n' "${action}" "${package}"
   fi
 done \
   < "${input_file}" \
-- 
cgit v1.2.3-54-g00ecf


From ac8df27bd6c262f917207a1959f26dff74d52aa0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 09:00:25 +0100
Subject: bin/get-package-updates: style

---
 bin/get-package-updates | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index fc66ec6..1ad5ce4 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -474,8 +474,10 @@ while [ -n "${black_listed_new}" ]; do
 done
 echo "${black_listed}" | \
   while read -r package; do
-    [ -n "${package}" ] && \
-      delete_package "${package}"
+    if [ -z "${package}" ]; then
+      continue
+    fi
+    delete_package "${package}"
   done
 
 sort -u "${work_dir}/deletion-list.new" | \
-- 
cgit v1.2.3-54-g00ecf


From 85f10d9ce376f8c921abd6f0266d5da62bb7c791 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 09:12:30 +0100
Subject: bin/get-package-updates: clarify todo

---
 bin/get-package-updates | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1ad5ce4..ffbd390 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -86,8 +86,12 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
-# TODO: New deletion-list packages should be recorded in the database,
-# too - including all binary information (especially dependencies!)
+# TODO: How should new deletion-list packages be handled?
+# - packages deleted upstream should be marked as to-be-deleted if
+#   existent and otherwise be ignored
+# - packages deleted due to black listing should be marked as black
+#   listed - and thus as to-be-deleted, but they should remain in the
+#   database after deletion!
 
 # delete_package package
 # mark $package for deletion
-- 
cgit v1.2.3-54-g00ecf


From 25c4fc4327cd66e86121dd2740772fb6cd6bcc20 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 09:16:17 +0100
Subject: bin/bootstrap-mysql: create `todos` and `todo_links`, too

---
 bin/bootstrap-mysql | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index ae51e25..a10b2d7 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -125,7 +125,14 @@ if [ ! "$1" = 'slim' ]; then
       ' build_assignment BIGINT :build_assignments' \
       ' date DATETIME' \
       ' reason SMALLINT :fail_reasons' \
-      ' log_file VARCHAR(512)'
+      ' log_file VARCHAR(512)' \
+      'todos MEDIUMINT' \
+      ' file VARCHAR(64)' \
+      ' line MEDIUMINT' \
+      ' description VARCHAR(512)'
+      'todo_links MEDIUMINT' \
+      ' dependent MEDIUMINT' \
+      ' depending_on MEDIUMINT'
   )
 
   {
-- 
cgit v1.2.3-54-g00ecf


From cc3d8e50cfb86a35cafd8b47f254c32ca68b929b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 12:07:33 +0100
Subject: bin/why-dont-you: build: read info from db

---
 bin/why-dont-you | 216 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 154 insertions(+), 62 deletions(-)

diff --git a/bin/why-dont-you b/bin/why-dont-you
index 830048b..83dc35f 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -19,68 +19,160 @@ case "${action}" in
 
   'build')
 
-    for pkg in "$@"; do
-      {
-        grep "^$(str_to_regex "${pkg}") " "${work_dir}/build-list" || \
-          >&2 printf '"%s" is not on the build list.\n' "${pkg}"
-      } | \
-        while read -r package git_revision mod_git_revision repository; do
-
-          if [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
-            printf '"%s" is locked by ' "$1"
-            sort -u < \
-              "${work_dir}/package-states/$1.$2.$3.$4.locked" \
-              sed '
-                :a
-                  $!{
-                    N
-                    s/\n/, /
-                    ba
-                  }
-                s/$/./
-              '
-            continue
-          fi
-          if [ -f "${work_dir}/package-states/$1.$2.$3.$4.blocked" ]; then
-            printf '"%s" is blocked: "' "${pkg}"
-            tr '[:space:]' ' ' < \
-              "${work_dir}/package-states/$1.$2.$3.$4.blocked" | \
-              sed '
-                s|  \+| |
-                s|^ ||
-                s| $||
-              '
-            printf '"\n'
-            continue
-          fi
-
-          unmet_dependencies=$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")
-          if [ -n "${unmet_dependencies}" ]; then
-            printf '"%s" has unmet dependencies:\n' "${package}"
-            echo "${unmet_dependencies}" | \
-              while read -r dep; do
-                grep -lxF "${dep}" "${work_dir}/package-infos/"*".builds" | \
-                  sed '
-                    s|^.*/||
-                    s|\(\.[^.]\+\)\{4\}||
-                  '
-              done | \
-              sort -u
-            printf '\n'
-
-            continue
-          fi
-
-          if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then
-            printf '"%s" is broken (%sx built), but would be built\n' \
-              "${pkg}" \
-              "$(wc -l < "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken")"
-          else
-            printf '"%s" would be built\n' "${pkg}"
-          fi
-        done
-
-    done
+    # shellcheck disable=SC2016
+    {
+      printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64));\n'
+      printf 'INSERT INTO `pkgbases` VALUES '
+      # shellcheck disable=SC2046
+      printf '(from_base64("%s")),' \
+        $(
+          printf '%s\n' "$@" | \
+            base64_encode_each
+        ) | \
+        sed 's/,$/;\n/'
+      # we select everything which is possibly of any interest:
+      # - id (to see if it actually is on the build-list
+      # - to_build.is_broken
+      # - failed_builds_count
+      # - to_build.is_blocked
+      # - deps.pkgbase (any dependency pending?)
+      # - build_slaves.name (is anyone building this?)
+      # - pkgbase
+      printf 'SELECT DISTINCT `to_build`.`ba_id`,'
+      printf 'If(`to_build`.`is_broken`,1,0),'
+      printf '('
+        printf 'SELECT count(*) FROM `failed_builds`'
+        printf 'WHERE `failed_builds`.`build_assignment`=`to_build`.`ba_id`'
+      printf ')'
+      printf ',replace(to_base64(`%s`.`%s`),"\\n","")' \
+        'to_build' 'is_blocked' \
+        'deps' 'pkgbase' \
+        'build_slaves' 'name' \
+        'pkgbases' 'pkgbase'
+      # at least one row for each given `pkgbase`
+      printf ' FROM `pkgbases`'
+      printf ' LEFT JOIN '
+        printf '('
+        # join the tables for the to-be-built packages:
+        # package_source, build_assignment, binary_package, repostory
+        printf 'SELECT DISTINCT `tb_ps`.`pkgbase`,`tb_bin`.`id` AS `bin_id`,`tb_ba`.`id` AS `ba_id`,`tb_ba`.`is_blocked`,`tb_ba`.`is_broken`'
+        printf ' FROM `package_sources` AS `tb_ps`'
+        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+          'build_assignments` AS `tb_ba' 'tb_ba'  'package_source'   'tb_ps' \
+          'binary_packages` AS `tb_bin'  'tb_bin' 'build_assignment' 'tb_ba' \
+          'repositories` AS `tb_rep'     'tb_bin' 'repository'       'tb_rep'
+        printf ' WHERE `tb_rep`.`name`="build-list"'
+        printf ') AS `to_build`'
+      printf ' ON `to_build`.`pkgbase`=`pkgbases`.`pkgbase`'
+      printf ' LEFT JOIN '
+        printf '('
+        # same join as above, but with different names - for the
+        # potential dependencies
+        printf 'SELECT DISTINCT `dep_ps`.`pkgbase`,`dependencies`.`dependent`'
+        printf ' FROM `package_sources` AS `dep_ps`'
+        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+          'build_assignments` AS `dep_ba' 'dep_ba'                   'package_source'   'dep_ps' \
+          'binary_packages` AS `dep_bin'  'dep_bin'                  'build_assignment' 'dep_ba' \
+          'repositories` AS `dep_rep'     'dep_bin'                  'repository'       'dep_rep' \
+          'install_target_providers'      'install_target_providers' 'package'          'dep_bin'
+        # starting from the line above, we have some additional joins,
+        # because we are interested in dependency relations to `to_build`
+        printf ' JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+          'dependency_types' 'dependencies' 'dependency_type' 'dependency_types'
+        printf ' WHERE `dep_rep`.`name`="build-list"'
+        printf ' AND `dependency_types`.`relevant_for_building`'
+        printf ') AS `deps`'
+      printf ' ON `deps`.`dependent`=`to_build`.`bin_id`'
+      # now we join with build slaves to see if someone builds this
+      printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`'
+      printf ';\n'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '
+        1d
+        y/\t/ /
+      ' | \
+      sort -k7,7 -k6,6 -k5,5 | \
+      sed '
+        / NULL \S\+$/ b multi-dep
+        :multi-slave
+          $!N
+          s/^\(\(\S\+ \)\{5\}\)\(\S\+\)\( \S\+\)\n\(\S\+ \)\{5\}\(\S\+\)\4/\1\3,\6\4/
+          t multi-slave
+          P
+          D
+        :multi-dep
+          / NULL\( \S\+\)\{3\}$/! b
+          $!N
+          s/^\(\(\S\+ \)\{4\}\)\(\S\+\)\(\( \S\+\)\{2\}\)\n\(\S\+ \)\{4\}\(\S\+\)\4/\1\3,\7\4/
+          t multi-dep
+          P
+          D
+      ' | \
+      sed '
+        s/NULL,//g
+      ' | \
+      while read -r id is_broken trials is_blocked dependency slave pkgbase; do
+#      printf '"%s" ' "$id" "$is_broken" "$trials" "$is_blocked" "$dependency" "$slave" "$pkgbase" >&2
+#      printf '\n' >&2
+        pkgbase=$(
+          printf '%s' "${pkgbase}" | \
+            base64 -d
+        )
+        if [ "${id}" = 'NULL' ]; then
+          >&2 printf '"%s" is not on the build list.\n' \
+            "${pkgbase}"
+          continue
+        fi
+        if [ "${slave}" != 'NULL' ]; then
+          # beware: A slave named "5BË" will look exactly like this!
+          printf '"%s" is locked by %s.\n' \
+            "${pkgbase}" \
+            "$(
+              printf '%s\n' "${slave}" | \
+                tr ',' '\n' | \
+                while read -r line; do
+                  printf '%s\n' "${line}" | \
+                    base64 -d
+                  printf ','
+                done | \
+                sed 's/,$//'
+            )"
+          continue
+        fi
+        if [ "${is_blocked}" != 'NULL' ]; then
+          # beware: A block-reason "5BË" will look exactly like this!
+          printf '"%s" is blocked: "%s".\n' \
+            "${pkgbase}" \
+            "$(
+              printf '%s' "${is_blocked}" | \
+                base64 -d
+            )"
+          continue
+        fi
+        if [ "${dependency}" != 'NULL' ]; then
+          printf '"%s" has unmet dependencies:\n' \
+            "${pkgbase}"
+          printf '%s\n' "${dependency}" | \
+            tr ',' '\n' | \
+            while read -r line; do
+              printf '  '
+              printf '%s\n' "${line}" | \
+                base64 -d
+              printf '\n'
+            done
+          continue
+        fi
+        if [ "${is_broken}" = '1' ]; then
+          printf '"%s" is broken (%sx built), but would be built.\n' \
+            "${pkgbase}" \
+            "${trials}"
+          continue
+        fi
+        printf '"%s" would be built.\n' \
+          "${pkgbase}"
+      done
 
   ;;
 
-- 
cgit v1.2.3-54-g00ecf


From 4a0fb5fa26056b7551f812f1e39d5e00a0c69c6e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 12:54:45 +0100
Subject: bin/return-assignment: read some stuff from the database

---
 bin/return-assignment | 45 ++++++++++++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index bf27232..aa79554 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -49,17 +49,38 @@ trap clean_up_lock_file EXIT
 if [ "$5" = 'ERROR' ]; then
 # the build failed on the build slave
 
-  if ! grep -qxF "$1 $2 $3 $4" "${work_dir}/build-list" ||
-    ! [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
-    >&2 echo 'Too late, package already outdated, I ignore this error report.'
-    exit 2
-  fi
-
-  # shellcheck disable=SC2154
-  if ! grep -qxF "${slave}" "${work_dir}/package-states/$1.$2.$3.$4.locked"; then
-    >&2 echo 'You do not build this package - move on.'
+  # shellcheck disable=SC2016
+  infos=$(
+    {
+      printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`'
+      printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+        'build_assignments'     'build_assignments'     'build_slaves'      'currently_building' \
+        'package_sources'       'package_sources'       'build_assignments' 'package_source' \
+        'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository' \
+        'binary_packages'       'build_assignments'     'binary_packages'   'build_assignment' \
+        'repositories'          'repositories'          'binary_packages'   'repository'
+      printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+        "$(
+          # shellcheck disable=SC2154
+          printf '%s' "${slave}" | \
+            base64 -w0
+        )"
+      printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+        'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
+        'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
+        'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+      printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+        "$(printf '%s' "$4" | base64 -w0)"
+      printf ' AND `repositories`.`name`="build-list"'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '1d'
+  )
+  if [ -z "${infos}" ]; then
+    >&2 echo 'You do not build this package (anymore) - move on.'
     exit 2
   fi
+  was_broken_before="${infos##* }"
 
   # save sent build logs
   saved_build_logs=$(
@@ -71,12 +92,6 @@ if [ "$5" = 'ERROR' ]; then
       '*.build-log.gz'
   )
 
-  if [ -f "${work_dir}/package-states/$1.$2.$3.$4.broken" ]; then
-    was_broken_before=true
-  else
-    was_broken_before=false
-  fi
-
   # shellcheck disable=SC2154
   echo "${slave}" >> \
     "${work_dir}/package-states/$1.$2.$3.$4.broken"
-- 
cgit v1.2.3-54-g00ecf


From 13d43b1f065af8c0a260b486e7b2fa9491e1ff33 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 13:04:15 +0100
Subject: bin/seed-build-list: also take known-packages from the database

---
 bin/seed-build-list | 43 +++++++++++++++++++++++--------------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 9a0123e..e17242c 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -133,28 +133,31 @@ mod_repo_rev=$(cat "${work_dir}/archlinux32.revision")
     generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}"
   done
 
-# shellcheck disable=SC2016
 {
-  printf 'SELECT '
-  printf '`package_sources`.`%s`,' \
-    'pkgbase' 'git_revision' 'mod_git_revision'
-  printf '`upstream_repositories`.`name`,`binary_packages`.`pkgname`'
-  printf ' FROM `binary_packages`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
-    'package_sources'       'package_sources'       'build_assignments' 'package_source' \
-    'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT '
+    printf '`package_sources`.`%s`,' \
+      'pkgbase' 'git_revision' 'mod_git_revision'
+    printf '`upstream_repositories`.`name`,`binary_packages`.`pkgname`'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
+      'package_sources'       'package_sources'       'build_assignments' 'package_source' \
+      'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '
+      1d
+      y/\t/ /
+    '
+  find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
+    sed '
+      s|^.*/||
+      s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 |
+    '
 } | \
-  ${mysql_command} --raw --batch | \
-  sed '
-    1d
-    y/\t/ /
-  ' > /dev/null # not yet working (the query works, but the entries are missing)
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
-  sed '
-    s|^.*/||
-    s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 |
-  ' | \
+  sort -u | \
   sort -k5,5 > \
   "${tmp_dir}/known-packages"
 
-- 
cgit v1.2.3-54-g00ecf


From 34c524246f2257b9cc94668fa07412ed44b09f40 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 14:26:12 +0100
Subject: bin/mysql-functions:
 mysql_repair_binary_packages_without_build_assignment started (and finished?)

---
 bin/bootstrap-mysql |  2 ++
 bin/mysql-functions | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index a10b2d7..116b6c7 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -587,3 +587,5 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
   } | \
     ${mysql_command}
 fi
+
+mysql_repair_binary_packages_without_build_assignment
diff --git a/bin/mysql-functions b/bin/mysql-functions
index d0861e7..1f2e882 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -771,3 +771,63 @@ mysql_query_select_pkgbase_and_revision() {
     'package_sources' 'package_sources' 'build_assignments' 'package_source' \
     'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
 }
+
+# mysql_repair_binary_packages_without_build_assignment
+# try to generate valid build assignments to binary packages without
+# a valid one yet
+mysql_repair_binary_packages_without_build_assignment() {
+  {
+    printf 'SELECT '
+    printf '`binary_packages`.`id`'
+    printf ',replace(to_base64(%s),"\\n","")' \
+      '`binary_packages`.`pkgname`' \
+      '`architectures`.`name`'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+    printf ' WHERE `binary_packages`.`build_assignment`<0'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '1d' | \
+    while read -r id pkgname arch; do
+      pkgname=$(
+        printf '%s' "${pkgname}" | \
+          base64 -d
+      )
+      pkgbase=$(
+        curl -Ss "$(
+          printf 'https://www.archlinux.org/packages/search/json/?name=%s' \
+            "${pkgname}"
+        )" | \
+          sed '
+            s/^.*"results":\s*\[//
+            s/}\s*,\s*{/\n/g
+          ' | \
+          grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \
+          tr ',' '\n' | \
+          grep '"pkgbase":' | \
+          cut -d'"' -f4
+      )
+      if [ -z "${pkgbase}" ]; then
+        >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}"
+        continue
+      fi
+      pkgbase=$(
+        printf '%s' "${pkgbase}" | \
+          base64 -w0
+      )
+      printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
+      printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0'
+      printf ' FROM `package_sources`'
+      printf ' JOIN `architectures`'
+      printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}"
+      printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}"
+      printf ' LIMIT 1;\n'
+      printf 'UPDATE `binary_packages`'
+      printf ' JOIN `build_assignments`'
+      printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+      printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+      printf ' WHERE `binary_packages`.`id`=%s' "${id}"
+      printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}"
+    done | \
+    ${mysql_command}
+}
-- 
cgit v1.2.3-54-g00ecf


From 29cd25770761c0cc6e056807789f32493470d550 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 16:27:05 +0100
Subject: bin/mysql-functions: add unknown packages whose pkgbase is
 identically found in the database

---
 bin/mysql-functions | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 1f2e882..3c02511 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -805,8 +805,21 @@ mysql_repair_binary_packages_without_build_assignment() {
           grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \
           tr ',' '\n' | \
           grep '"pkgbase":' | \
-          cut -d'"' -f4
+          cut -d'"' -f4 | \
+          sort -u | \
+          head -n1
       )
+      if [ -z "${pkgbase}" ] && \
+        {
+          printf 'SELECT count(*) FROM `package_sources`'
+          printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
+            "$(printf '%s' "${pkgname}" | base64 -w0)"
+        } | \
+          ${mysql_command} --raw --batch | \
+          sed '1d' | \
+          grep -vxF '0'; then
+        pkgbase="${pkgname}"
+      fi
       if [ -z "${pkgbase}" ]; then
         >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}"
         continue
-- 
cgit v1.2.3-54-g00ecf


From 6af72dc8648dd3ed3657c3c8b2cd0e00be5b5c9b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 16:33:00 +0100
Subject: bin/mysql-functions: small bugfix - grep should be silenced :-)

---
 bin/mysql-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 3c02511..1e78751 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -817,7 +817,7 @@ mysql_repair_binary_packages_without_build_assignment() {
         } | \
           ${mysql_command} --raw --batch | \
           sed '1d' | \
-          grep -vxF '0'; then
+          grep -qvxF '0'; then
         pkgbase="${pkgname}"
       fi
       if [ -z "${pkgbase}" ]; then
-- 
cgit v1.2.3-54-g00ecf


From d9923cbe074356559b481bf6e61d18cba09a878d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Feb 2018 16:48:51 +0100
Subject: bin/mysql-functions: fix comment

---
 bin/mysql-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 1e78751..6fb105f 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -194,7 +194,7 @@ mysql_add_install_target() {
 # mysql_generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
 # if sub_pkgrel should be determined automatically and the package is on the build-list
 # and
-# mysql_generate_package_metadata $sub_pkgrel $package $current_repository $git_revision $mod_git_revision $repository
+# mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository
 # or
 # mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository
 # if $sub_pkgrel should be forced and the package is currently in $current_repository
-- 
cgit v1.2.3-54-g00ecf


From 8aca660449fde7a425e8d7b0af49955ec90476fc Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 09:11:50 +0100
Subject: bin/get-package-updates: add TODO

---
 bin/get-package-updates | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index ffbd390..7b394c0 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -13,6 +13,9 @@
 
 # TODO: read information from database
 
+# TODO: correctly handle if pkgbase of a split package is renamed, e.g.:
+# $a -> ($a,$b)  ==>  $b -> ($a,$b)
+
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
-- 
cgit v1.2.3-54-g00ecf


From 87c2fd5fdc4030ea86f7ef075f469b73ed0caf5f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 09:41:29 +0100
Subject: web-scripts/to-delete.php new

---
 web-scripts/to-delete.php | 62 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 web-scripts/to-delete.php

diff --git a/web-scripts/to-delete.php b/web-scripts/to-delete.php
new file mode 100644
index 0000000..3d732a1
--- /dev/null
+++ b/web-scripts/to-delete.php
@@ -0,0 +1,62 @@
+<html>
+<head>
+<title>List of packages to be deleted</title>
+<link rel="stylesheet" type="text/css" href="/static/style.css">
+</head>
+<body>
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`repositories`.`name` AS `repo`," .
+  "`is_there`.`pkgname`," .
+  "`is_there`.`epoch`," .
+  "`is_there`.`pkgver`," .
+  "`is_there`.`pkgrel`," .
+  "`is_there`.`sub_pkgrel`," .
+  "`architectures`.`name` AS `arch` " .
+  "FROM `binary_packages`AS `is_there` " .
+  "JOIN `binary_packages` AS `to_delete` ON `to_delete`.`pkgname`=`is_there`.`pkgname` " .
+  "JOIN `architectures` ON `is_there`.`architecture`=`architectures`.`id` " .
+  "JOIN `repositories` ON `is_there`.`repository`=`repositories`.`id` " .
+  "WHERE `to_delete`.`repository`=10 " .
+  "AND NOT `is_there`.`repository` IN (4,9,10)"
+);
+if ($result -> num_rows > 0) {
+
+  $count = 0;
+
+  while ($row = $result->fetch_assoc()) {
+    $rows[$count] =
+      $row["repo"] . "/" .
+      $row["pkgname"] . "-";
+    if ($row["epoch"] != "0")
+      $rows[$count] =
+        $rows[$count] .
+        $row["epoch"] . ":";
+    $rows[$count] =
+      $rows[$count] .
+      $row["pkgver"] . "-" .
+      $row["pkgrel"] . "." .
+      $row["sub_pkgrel"] . "-" .
+      $row["arch"] . ".pkg.tar.xz";
+    $count++;
+  }
+
+  sort($rows);
+
+  foreach ($rows as $row) {
+    print $row."<br>\n";
+  }
+} else {
+  print "No packages are to be deleted.\n";
+}
+
+?>
+</body>
+</html>
-- 
cgit v1.2.3-54-g00ecf


From 13b337ad76ae67057d64fca920d23a5389ae2c0e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 09:49:08 +0100
Subject: web-scripts/todos.php: display "$" correctly

---
 web-scripts/todos.php | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/web-scripts/todos.php b/web-scripts/todos.php
index 123fa35..e4ab2c9 100644
--- a/web-scripts/todos.php
+++ b/web-scripts/todos.php
@@ -48,9 +48,12 @@ if ($result -> num_rows > 0) {
     $edges=$edges . "\"" . $link["depending_on"] . "\" -> \"" . $link["dependent"] . "\";\n";
 }
 
+$knots = str_replace("\$","\\\$",$knots);
+$edges = str_replace("\$","\\\$",$edges);
+
 header ("Content-type: image/png");
 passthru(
-  "dot -Tpng -o/dev/stdout /dev/stdin <<EOF\n".
+  "dot -Tpng -o/dev/stdout /dev/stdin <<EOF\n" .
   "digraph dependencies {\n" .
   "rankdir=LR;\n" .
   "fontname=dejavu;\n" .
-- 
cgit v1.2.3-54-g00ecf


From 0a1661c353b65730bb905a17a165185dfbb32577 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 10:34:28 +0100
Subject: bin/bootstrap-mysql, bin/mysql-functions: new column
 `is_to_be_deleted` in `binary_packages`

---
 bin/bootstrap-mysql | 5 +++--
 bin/mysql-functions | 8 +++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 116b6c7..5acc2e2 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -61,6 +61,7 @@ if [ ! "$1" = 'slim' ]; then
       ' sub_pkgrel MEDIUMINT' \
       ' has_issues BIT' \
       ' is_tested BIT' \
+      ' is_to_be_deleted BIT' \
       ' pkgname VARCHAR(64)' \
       ' architecture SMALLINT :architectures' \
       ' UNIQUE content build_assignment sub_pkgrel pkgname architecture repository' \
@@ -562,7 +563,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
       s/,$/;/
     ' "${tmp_dir}/new-stable-packages"
     # shellcheck disable=SC2016
-    printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`)'
+    printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)'
     printf ' SELECT '
     # shellcheck disable=SC2016
     printf '`stable_packages`.`%s`,' \
@@ -576,7 +577,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
       'architectures' \
       'repositories'
     # shellcheck disable=SC2016
-    printf -- '-`build_assignment`,0,1 FROM `stable_packages`'
+    printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`'
     # shellcheck disable=SC2016
     printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \
       'repositories' 'repository' 'repositories' \
diff --git a/bin/mysql-functions b/bin/mysql-functions
index 6fb105f..c576260 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -90,10 +90,11 @@ mysql_add_binary_package() {
   {
     printf 'INSERT IGNORE INTO binary_packages'
     printf ' ('
-    printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture'
+    printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted'
     printf ') SELECT'
     printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}"
     printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures'
+    printf ' 0, 0, 0'
     printf ' FROM'
     printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures'
     printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
@@ -470,7 +471,8 @@ mysql_generate_package_metadata() {
             'pkgname' \
             'sub_pkgrel' \
             'has_issues' \
-            'is_tested'
+            'is_tested' \
+            'is_to_be_deleted'
           printf ') SELECT '
           printf '`%s`.`id`,' \
             'build_assignments' \
@@ -481,7 +483,7 @@ mysql_generate_package_metadata() {
             "${pkgver}" \
             "${pkgrel}" \
             "${pkgname}"
-          printf '%s,0,0 FROM' \
+          printf '%s,0,0,0 FROM' \
             "${sub_pkgrel}"
           printf ' `%s` JOIN' \
             'repositories' \
-- 
cgit v1.2.3-54-g00ecf


From 668b94e67649d45d3bd7dacf075dab14428ce24b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 10:35:33 +0100
Subject: bin/get-package-updates: use new column `is_to_be_deleted` in
 `binary_packages`

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 7b394c0..f451991 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -110,7 +110,7 @@ delete_package() {
       'build_assignments' 'build_assignments' 'build_assignment'
     printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
       'package_sources' 'package_sources' 'build_assignments' 'package_source'
-    printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
+    printf ' SET `binary_packages`.`is_to_be_deleted`=1'
     printf ' WHERE `repositories`.`name`="build-list"'
     printf ' AND `package_sources`.`pkgbase`=from_base64("%s");' \
       "$(printf '%s' "$1" | base64 -w0)"
-- 
cgit v1.2.3-54-g00ecf


From 057caf194aa7a9ecd49fba424d34065190304608 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 10:41:33 +0100
Subject: bin/build-master-status-from-mysql: is_to_be_deleted should be
 considered on the deletion-list, too

---
 bin/build-master-status-from-mysql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 8e828f7..408ef1e 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -40,7 +40,8 @@ sort "${work_dir}/build-list" > \
     'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
     'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \
     'repositories' 'binary_packages' 'repository' 'repositories'
-  printf ' WHERE `repositories`.`name`="deletion-list";\n'
+  printf ' WHERE `repositories`.`name`="deletion-list"'
+  printf ' OR `binary_packages`.`is_to_be_deleted`;\n'
 } | \
   ${mysql_command} --batch | \
   sed '
-- 
cgit v1.2.3-54-g00ecf


From ae5771a8302c7f2d9353911363ffcb3fab14f9ee Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 10:50:32 +0100
Subject: web-scripts/to-delete.php: use `is_to_be_deleted` column from
 `binary_packages`

---
 web-scripts/to-delete.php | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/web-scripts/to-delete.php b/web-scripts/to-delete.php
index 3d732a1..dfa5a39 100644
--- a/web-scripts/to-delete.php
+++ b/web-scripts/to-delete.php
@@ -12,20 +12,19 @@ if ($mysql->connect_error) {
 }
 
 $result = $mysql -> query(
-  "SELECT DISTINCT " .
+  "SELECT " .
   "`repositories`.`name` AS `repo`," .
-  "`is_there`.`pkgname`," .
-  "`is_there`.`epoch`," .
-  "`is_there`.`pkgver`," .
-  "`is_there`.`pkgrel`," .
-  "`is_there`.`sub_pkgrel`," .
+  "`binary_packages`.`pkgname`," .
+  "`binary_packages`.`epoch`," .
+  "`binary_packages`.`pkgver`," .
+  "`binary_packages`.`pkgrel`," .
+  "`binary_packages`.`sub_pkgrel`," .
   "`architectures`.`name` AS `arch` " .
-  "FROM `binary_packages`AS `is_there` " .
-  "JOIN `binary_packages` AS `to_delete` ON `to_delete`.`pkgname`=`is_there`.`pkgname` " .
-  "JOIN `architectures` ON `is_there`.`architecture`=`architectures`.`id` " .
-  "JOIN `repositories` ON `is_there`.`repository`=`repositories`.`id` " .
-  "WHERE `to_delete`.`repository`=10 " .
-  "AND NOT `is_there`.`repository` IN (4,9,10)"
+  "FROM `binary_packages` " .
+  "JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id` " .
+  "JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id` " .
+  "WHERE `binary_packages`.`is_to_be_deleted` " .
+  "AND NOT `repositories`.`name` IN (\"build-support\",\"build-list\",\"deletion-list\")"
 );
 if ($result -> num_rows > 0) {
 
-- 
cgit v1.2.3-54-g00ecf


From e2ca9121e30856651afef3c6396175dca1f6bd91 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 11:35:03 +0100
Subject: bin/return-assignment: fix whitespace in mysql response

---
 bin/return-assignment | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index aa79554..01a20c8 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -74,7 +74,10 @@ if [ "$5" = 'ERROR' ]; then
       printf ' AND `repositories`.`name`="build-list"'
     } | \
       ${mysql_command} --raw --batch | \
-      sed '1d'
+      sed '
+        1d
+        y/\t/ /
+      '
   )
   if [ -z "${infos}" ]; then
     >&2 echo 'You do not build this package (anymore) - move on.'
-- 
cgit v1.2.3-54-g00ecf


From aa1158a7f261100312057c94bfb40553c52296a6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 11:44:27 +0100
Subject: bin/show-dependencies: read some information from db

---
 bin/show-dependencies | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index ee7db19..450120f 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -9,21 +9,35 @@
 
 if [ $# -eq 0 ]; then
   broken=$(
-    find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
-      sed 's|\(\.[^.]\+\)\{4\}$||' | \
+    # shellcheck disable=SC2016
+    {
+      printf 'SELECT `package_sources`.`pkgbase`'
+      printf ' FROM `package_sources`'
+      printf ' JOIN `build_assignments` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+      printf ' WHERE `build_assignments`.`is_broken`'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '1d' | \
       sort -u
   )
   new_sum='x'
   sum=''
   tmp_dir=$(mktemp -d 'tmp.show-dependencies.0.XXXXXXXXXX' --tmpdir)
   trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-  find "${work_dir}/package-infos" -maxdepth 1 -name '*.builds' -exec \
-    grep -HF '' {} \; | \
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`binary_packages`.`pkgname`'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'build_assignments' 'build_assignments' 'binary_packages'   'build_assignment' \
+      'package_sources'   'package_sources'   'build_assignments' 'package_source'
+    printf ' ORDER BY `binary_packages`.`pkgname`'
+  } | \
+    ${mysql_command} --raw --batch | \
     sed '
-      s|^.*/||
-      s|\(\.[^.]\+\)\{4\}:| |
-    ' | \
-    sort -k2,2 > \
+      1d
+      y/\t/ /
+    ' > \
     "${tmp_dir}/builds"
   while ! [ "${new_sum}" = "${sum}" ]; do
     sum="${new_sum}"
-- 
cgit v1.2.3-54-g00ecf


From 3e99365e5fc1c608e60c0465eba520806060413b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 12:00:31 +0100
Subject: bin/show-dependencies: ORDER BY does not sort like "sort" :-/

---
 bin/show-dependencies | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index 450120f..23e8e5f 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -32,12 +32,14 @@ if [ $# -eq 0 ]; then
       'build_assignments' 'build_assignments' 'binary_packages'   'build_assignment' \
       'package_sources'   'package_sources'   'build_assignments' 'package_source'
     printf ' ORDER BY `binary_packages`.`pkgname`'
+    # Why can't mysql order in the same way "sort" does!
   } | \
     ${mysql_command} --raw --batch | \
     sed '
       1d
       y/\t/ /
-    ' > \
+    ' | \
+    sort -k2,2 > \
     "${tmp_dir}/builds"
   while ! [ "${new_sum}" = "${sum}" ]; do
     sum="${new_sum}"
-- 
cgit v1.2.3-54-g00ecf


From 86823d203d23536386d7c01356e2a5a6df810adf Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 13:48:56 +0100
Subject: bin/get-package-updates: build-list packages should be "deleted"
 immediately

---
 bin/get-package-updates | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index f451991..180e0ae 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -104,15 +104,26 @@ delete_package() {
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
   # shellcheck disable=SC2016
   {
+    # packages from the build-list go straight to the deletion-list
     printf 'UPDATE `binary_packages`'
     printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
       'repositories' 'repositories' 'repository' \
       'build_assignments' 'build_assignments' 'build_assignment'
     printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
       'package_sources' 'package_sources' 'build_assignments' 'package_source'
-    printf ' SET `binary_packages`.`is_to_be_deleted`=1'
+    printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
     printf ' WHERE `repositories`.`name`="build-list"'
-    printf ' AND `package_sources`.`pkgbase`=from_base64("%s");' \
+    printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \
+      "$(printf '%s' "$1" | base64 -w0)"
+    # other packages are marked as `is_to_be_deleted`
+    printf 'UPDATE `binary_packages`'
+    printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
+      'repositories' 'repositories' 'repository' \
+      'build_assignments' 'build_assignments' 'build_assignment'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'package_sources' 'package_sources' 'build_assignments' 'package_source'
+    printf ' SET `binary_packages`.`is_to_be_deleted`=1'
+    printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \
       "$(printf '%s' "$1" | base64 -w0)"
   } | \
     ${mysql_command}
-- 
cgit v1.2.3-54-g00ecf


From 31dfe057f9b5667991f3c7809a13a87485f32ac6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 20:52:56 +0100
Subject: bin/ping-from-slave new

---
 bin/ping-from-slave     | 11 +++++++++++
 bin/slave-build-connect |  7 ++++---
 2 files changed, 15 insertions(+), 3 deletions(-)
 create mode 100755 bin/ping-from-slave

diff --git a/bin/ping-from-slave b/bin/ping-from-slave
new file mode 100755
index 0000000..4013269
--- /dev/null
+++ b/bin/ping-from-slave
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# should be called periodically on the build-master from the slaves to
+#  - report any update on the build process
+#  - show that the build is still running
+#  - get notified by the build master if the build is not necessary anymore
+
+# shellcheck source=conf/default.conf
+. "${0%/*}/../conf/default.conf"
+
+# TODO: acutally do something here :-)
diff --git a/bin/slave-build-connect b/bin/slave-build-connect
index 2ca72a3..119592e 100755
--- a/bin/slave-build-connect
+++ b/bin/slave-build-connect
@@ -3,11 +3,12 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-if [ "${SSH_ORIGINAL_COMMAND%% *}" = "get-assignment" ] || \
-  [ "${SSH_ORIGINAL_COMMAND%% *}" = "return-assignment" ]; then
+if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \
+  [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xreturn-assignment' ] || \
+  [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xping-from-slave' ]; then
 
   # small check to prevent some shell-injections
-  if echo "${SSH_ORIGINAL_COMMAND}" | \
+  if printf '%s\n' "${SSH_ORIGINAL_COMMAND}" | \
     grep -q '[^-a-zA-Z0-9.+_ ]'; then
 
     >&2 printf 'Invalid command: "%s".\n' "${SSH_ORIGINAL_COMMAND}"
-- 
cgit v1.2.3-54-g00ecf


From 4d289c1d38a240b6e7f5a2a3c805bbb82c459ae9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Feb 2018 21:24:43 +0100
Subject: bin/show-dependencies: build-list packages should have higher
 priority than staging packages

---
 bin/show-dependencies | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index 23e8e5f..0a7a1fd 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -218,7 +218,7 @@ for target_package in "$@"; do
       sed 's|\(\.[^.]\+\)\{4\}$||' | \
       sort -u | \
       tee "${tmp_dir}/done-packages" | \
-      sed 's|^|0 staging-package |' > \
+      xargs -r printf '4 staging-package %s\n' > \
       "${tmp_dir}/knots"
 
     {
@@ -357,7 +357,7 @@ for target_package in "$@"; do
   } | \
     sort | \
     uniq -d | \
-    xargs -r printf '4 deletion-list-package %s\n' >> \
+    xargs -r printf '5 deletion-list-package %s\n' >> \
     "${tmp_dir}/knots"
 
   {
@@ -369,7 +369,7 @@ for target_package in "$@"; do
   } | \
     sort | \
     uniq -d | \
-    xargs -r printf '5 split-package %s\n' >> \
+    xargs -r printf '6 split-package %s\n' >> \
     "${tmp_dir}/knots"
 
   {
@@ -380,7 +380,7 @@ for target_package in "$@"; do
   } | \
     sort | \
     uniq -d | \
-    xargs -r printf '6 package %s\n' >> \
+    xargs -r printf '7 package %s\n' >> \
     "${tmp_dir}/knots"
 
   {
@@ -392,7 +392,7 @@ for target_package in "$@"; do
   } | \
     sort | \
     uniq -d | \
-    xargs -r printf '7 group %s\n' >> \
+    xargs -r printf '8 group %s\n' >> \
     "${tmp_dir}/knots"
 
   tr ' ' '\n' < \
-- 
cgit v1.2.3-54-g00ecf


From eebf4ff1ff5f243645999857e021d7749c45090d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Feb 2018 09:36:19 +0100
Subject: bin/build-master-status-from-mysql: get shared build-list lock

---
 bin/build-master-status-from-mysql | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 408ef1e..7a3d7e6 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -10,6 +10,10 @@
 tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
+# always block if locked
+exec 9> "${build_list_lock_file}"
+flock -s 9
+
 # shellcheck disable=SC2016
 {
   printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`'
-- 
cgit v1.2.3-54-g00ecf


From d7e35686716933cbfaac6b900d7d2568063b7e9f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Feb 2018 16:00:33 +0100
Subject: bin/build-master-status-from-mysql: include build-dependencies

---
 bin/build-master-status-from-mysql | 39 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 7a3d7e6..3cacfa1 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -151,6 +151,41 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   sort -u > \
   "${tmp_dir}/testing.file"
 
+# shellcheck disable=SC2016
+{
+  printf 'SELECT '
+  printf '`package_sources`.`%s`,' \
+    'pkgbase' \
+    'git_revision' \
+    'mod_git_revision'
+  printf '`upstream_repositories`.`name`,`install_targets`.`name`'
+  printf ' FROM `binary_packages`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'dependencies'     'binary_packages'  'dependencies' 'dependent' \
+    'dependency_types' 'dependency_types' 'dependencies' 'dependency_type'
+  printf ' AND `dependency_types`.`relevant_for_building`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'install_targets'       'install_targets'       'dependencies'      'depending_on' \
+    'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
+    'package_sources'       'package_sources'       'build_assignments' 'package_source' \
+    'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+} | \
+  ${mysql_command} --raw --batch | \
+  sed '
+    1d
+    y/\t/ /
+  ' | \
+  sort -u > \
+  "${tmp_dir}/build-dependencies.mysql"
+find "${work_dir}/package-infos" -name '*.build-depends' \
+  -exec grep -HF '' {} \; | \
+  sed '
+    s,^.*/,,
+    s/\.\([^.]\+\)\.\([^.]\+\)\.\([^.:]\+\)\.build-depends:\([^:]\+\)$/ \1 \2 \3 \4/
+  ' | \
+  sort -u > \
+  "${tmp_dir}/build-dependencies.file"
+
 {
   mysql_sanity_check || true
   diff -u \
@@ -173,6 +208,10 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     "${tmp_dir}/testing.file" \
     "${tmp_dir}/testing.mysql" || \
     true
+  diff -u \
+    "${tmp_dir}/build-dependencies.file" \
+    "${tmp_dir}/build-dependencies.mysql" || \
+    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
-- 
cgit v1.2.3-54-g00ecf


From 62526023ab4bb6dda4e8189ca74969497bb3ef09 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Feb 2018 21:06:52 +0100
Subject: bin/build-master-status-from-mysql: do not block on lock failure

---
 bin/build-master-status-from-mysql | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 3cacfa1..6ed2e84 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -12,7 +12,10 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
 # always block if locked
 exec 9> "${build_list_lock_file}"
-flock -s 9
+if ! flock -n -s 9; then
+  >&2 echo 'Mysql-Sanity check skipped, cannot acquire lock.'
+  exit
+fi
 
 # shellcheck disable=SC2016
 {
-- 
cgit v1.2.3-54-g00ecf


From 9ec11f6a3948a1840dc25f7186d688e6634096ea Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Feb 2018 21:44:08 +0100
Subject: bin/build-master-status-from-mysql: ignore build-dependencies which
 are in the database but not in the files (some old dependencies will get
 removed from the files, but not (yet) from the database)

---
 bin/build-master-status-from-mysql | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 6ed2e84..16afb9f 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -180,12 +180,15 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   ' | \
   sort -u > \
   "${tmp_dir}/build-dependencies.mysql"
-find "${work_dir}/package-infos" -name '*.build-depends' \
-  -exec grep -HF '' {} \; | \
-  sed '
-    s,^.*/,,
-    s/\.\([^.]\+\)\.\([^.]\+\)\.\([^.:]\+\)\.build-depends:\([^:]\+\)$/ \1 \2 \3 \4/
-  ' | \
+{
+  cat "${tmp_dir}/build-dependencies.mysql"
+  find "${work_dir}/package-infos" -name '*.build-depends' \
+    -exec grep -HF '' {} \; | \
+    sed '
+      s,^.*/,,
+      s/\.\([^.]\+\)\.\([^.]\+\)\.\([^.:]\+\)\.build-depends:\([^:]\+\)$/ \1 \2 \3 \4/
+    '
+} | \
   sort -u > \
   "${tmp_dir}/build-dependencies.file"
 
-- 
cgit v1.2.3-54-g00ecf


From fd4fbd34ec608f22394a6367ced9bdcf3b13ce6a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 10:46:37 +0100
Subject: bin/get-package-updates: removing a package from the deletion list
 should remove is_to_be_deleted markers, too

---
 bin/get-package-updates | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 180e0ae..8c758c5 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -323,6 +323,18 @@ done | \
               base64 -w0
           )"
         printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n'
+        # remove is-to-be-deleted marker from old binary packages
+        printf 'UPDATE `binary_packages`'
+        printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+          'build_assignments' 'build_assignments' 'binary_packages'   'build_assignment' \
+          'package_sources'   'package_sources'   'build_assignments' 'package_source' \
+          'repositories'      'repositories'      'binary_packages'   'repository'
+        printf ' SET `is_to_be_deleted`=0'
+        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")\;' \
+          "$(
+            printf '%s' "${package}" | \
+              base64 -w0
+          )"
       } | \
         ${mysql_command}
       mysql_generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
-- 
cgit v1.2.3-54-g00ecf


From ac7e8a2881d5d76c822d761d5f613bf2a3b20187 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 10:57:58 +0100
Subject: bin/ii-connect: specify irc host (how did that work before???)

---
 bin/ii-connect | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/ii-connect b/bin/ii-connect
index 54e1eb0..cfe5043 100755
--- a/bin/ii-connect
+++ b/bin/ii-connect
@@ -8,7 +8,7 @@
 # start ii if it is not running
 if ! pgrep -x ii > /dev/null; then
   rm -rf --one-file-system "${irc_dir}"
-  screen -S ii -d -m ii -f buildmaster -n buildmaster
+  screen -S ii -d -m ii -s irc.freenode.net -f buildmaster -n buildmaster
   sleep 10
 fi
 
-- 
cgit v1.2.3-54-g00ecf


From 5c5b3b2ef97347ba70458ea02989e838b5c76393 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 11:31:14 +0100
Subject: bin/get-package-updates: fix typo in mysql query

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 8c758c5..e57d72b 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -330,7 +330,7 @@ done | \
           'package_sources'   'package_sources'   'build_assignments' 'package_source' \
           'repositories'      'repositories'      'binary_packages'   'repository'
         printf ' SET `is_to_be_deleted`=0'
-        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")\;' \
+        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \
           "$(
             printf '%s' "${package}" | \
               base64 -w0
-- 
cgit v1.2.3-54-g00ecf


From 54f1d508044492fdd4410defbc94963bd0608d63 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 14:10:07 +0100
Subject: bin/get-package-updates: take dependency information for black
 listing from database

---
 bin/get-package-updates | 226 +++++++++++++++++++-----------------------------
 1 file changed, 87 insertions(+), 139 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index e57d72b..4fcefd9 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -367,18 +367,21 @@ echo 'apply blacklisting'
 black_listed=''
 black_listed_new=$(
   {
-    git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \
-      tar -Ox 'blacklist' | \
-      sed '
-        s/\s*#.*$//
-        /^\s*$/d
-      '
-    if [ -n "${test_exclusion}" ]; then
-      echo "${test_exclusion}"
-    fi
+    {
+      git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \
+        tar -Ox 'blacklist' | \
+        sed '
+          s/\s*#.*$//
+          /^\s*$/d
+        '
+      if [ -n "${test_exclusion}" ]; then
+        echo "${test_exclusion}"
+      fi
+    } | \
+      base64_encode_each
     # shellcheck disable=SC2016
     {
-      printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
+      printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")'
       printf ' FROM `package_sources`'
       printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
     } | \
@@ -388,120 +391,91 @@ black_listed_new=$(
     sort -u
 )
 
-find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' -printf '%f\n' | \
-  grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.builds$' | \
-  sed '
-    s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$|\2 \3 \4 \1|
-  ' | \
-  sort -u | \
-  sort -k4,4 > \
-  "${tmp_dir}/packages"
-
-{
-  uniq -f3 -u "${tmp_dir}/packages" | \
-    awk '{print $4 " " $4 "." $1 "." $2 "." $3}'
-  uniq -f3 -D "${tmp_dir}/packages" | \
-    uniq -f3 --group=append | \
-    while read -r rev mod_rev repo pkg; do
-      if [ -z "${rev}" ] && \
-        [ -z "${mod_rev}" ] && \
-        [ -z "${repo}" ] && \
-        [ -z "${pkg}" ]; then
-        # shellcheck disable=SC2031
-        printf '%s %s.%s.%s.%s\n' \
-          "${opkg}" \
-          "${opkg}" \
-          "$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${revs} | \
-              sort -u | \
-              find_newest_of_git_revisions
-          )" \
-          "$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${mod_revs} | \
-              sort -u | \
-              find_newest_of_git_revisions
-          )" \
-          "${orepo}" | \
-          grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+$'
-        revs=''
-        mod_revs=''
-        continue
-      fi
-      revs=$(
-        printf '%s\n' \
-          ${revs} \
-          "${rev}"
-      )
-      mod_revs=$(
-        printf '%s\n' \
-          ${mod_revs} \
-          "${mod_rev}"
-      )
-      opkg="${pkg}"
-      orepo="${repo}"
-    done
-} | \
-  sort -k1,1 > \
-  "${tmp_dir}/newest-revisions"
-
-sed '
-  s|^\S\+ |'"${work_dir}"'/package-infos/|
-  s|$|.builds|
-' "${tmp_dir}/newest-revisions" | \
-  xargs -r cat | \
-  sort | \
-  uniq -c > \
-  "${work_dir}/built-packages"
-
-sed '
-  s|^\S\+ |'"${work_dir}"'/package-infos/|
-  s|$|.build-depends|
-' "${tmp_dir}/newest-revisions" | \
-  xargs -r grep -HF '' | \
-  sed '
-    s|\(\.[^.]\+\)\{4\}:\([^:]\+\)$| \2|
-    s|^.*/||
-  ' | \
-  sort -k2,2 > \
-  "${work_dir}/newest-dependencies"
-
 while [ -n "${black_listed_new}" ]; do
   black_listed=$(
-    # shellcheck disable=SC2086
-    printf '%s\n' ${black_listed} ${black_listed_new} | \
+    printf '%s\n' "${black_listed}" "${black_listed_new}" | \
+      grep -vxF '' | \
       sort -u
   )
   black_listed_new=$(
     {
-      # shellcheck disable=SC2086
-      printf '%s\n' ${black_listed} | \
-        sort -k1,1 | \
-        join -j1 -o 2.2 - "${tmp_dir}/newest-revisions" | \
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.builds|
-        ' | \
-        xargs -r cat | \
-        sort | \
-        uniq -c
-      cat "${work_dir}/built-packages"
+      printf '%s\n' "${black_listed}" "${black_listed}"
+
+      # shellcheck disable=SC2016
+      {
+        printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
+        printf 'INSERT INTO `bl` (`pkgbase`) VALUES '
+        printf '(from_base64("%s")),\n' "${black_listed}" | \
+          sort -u | \
+          sed '
+            $ s/,$/;/
+          '
+        printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")'
+        printf ' FROM `package_sources` AS `a_ps`'
+        printf ' JOIN `build_assignments` AS `a_ba` ON `a_ba`.`package_source`=`a_ps`.`id`'
+        printf ' JOIN `binary_packages` AS `a_bp` ON `a_bp`.`build_assignment`=`a_ba`.`id`'
+        printf ' JOIN `dependencies` ON `dependencies`.`dependent`=`a_bp`.`id`'
+        printf ' WHERE NOT EXISTS ('
+          printf ' SELECT *'
+          printf ' FROM `install_target_providers`'
+          printf ' WHERE NOT EXISTS ('
+            printf ' SELECT *'
+            printf ' FROM `bl`'
+            printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
+            printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`'
+            printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`'
+            printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
+            printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+          printf ')'
+        printf ');\n'
+      } | \
+        ${mysql_command} --raw --batch | \
+        sed '1d' | \
+        sort -u
     } | \
-      sort | \
-      uniq -d | \
-      awk '{print $2}' | \
-      sort -k1,1 | \
-      join -1 1 -2 2 -o 2.1 - "${work_dir}/newest-dependencies" | \
-      sort -u
-  )
-  black_listed_new=$(
-    # shellcheck disable=SC2086
-    printf '%s\n' ${black_listed} ${black_listed} ${black_listed_new} | \
+      grep -vxF '' | \
       sort | \
       uniq -u
   )
 done
+
+black_listed=$(
+  printf '%s\n' "${black_listed}" | \
+  while read -r line; do
+    printf '%s' \
+      "${line}" | \
+       base64 -d
+    printf '\n'
+  done
+)
+
+deletion_list_count=$(
+  # shellcheck disable=SC2086
+  printf '%s\n' ${black_listed} | \
+    wc -l
+)
+if [ "${deletion_list_count}" -gt 1000 ]; then
+  >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
+    "${deletion_list_count}"
+  if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
+    {
+      printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
+      for repo_name in ${repo_names}; do
+        if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then
+          printf '%s: %s -> %s\n' \
+            "${repo_name}" \
+            "$(cat "${work_dir}/${repo_name}.revision")" \
+            "$(cat "${work_dir}/${repo_name}.revision.new")"
+        fi
+      done
+    } | \
+      tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
+      sponge "${irc_dir}/#archlinux-ports/in"
+  fi
+  exit 3
+fi
+rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
+
 echo "${black_listed}" | \
   while read -r package; do
     if [ -z "${package}" ]; then
@@ -599,32 +573,6 @@ fi
 # update loop list in database
 mysql_find_build_assignment_loops
 
-deletion_list_count=$(
-  wc -l < \
-    "${work_dir}/deletion-list.new"
-)
-if [ "${deletion_list_count}" -gt 1000 ]; then
-  >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
-    "${deletion_list_count}"
-  if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
-    {
-      printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
-      for repo_name in ${repo_names}; do
-        if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then
-          printf '%s: %s -> %s\n' \
-            "${repo_name}" \
-            "$(cat "${work_dir}/${repo_name}.revision")" \
-            "$(cat "${work_dir}/${repo_name}.revision.new")"
-        fi
-      done
-    } | \
-      tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
-      sponge "${irc_dir}/#archlinux-ports/in"
-  fi
-  exit 3
-fi
-rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
-
 # Move the .new-files to the actual files
 
 rm -rf --one-file-system "${work_dir}/build-list.loops"
-- 
cgit v1.2.3-54-g00ecf


From 3dd56fb3f3ce073827b0d168859520630dd77894 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 16:18:04 +0100
Subject: bin/mysql-functions: more cleanup and checks

---
 bin/mysql-functions | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index c576260..ea3ea78 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -622,6 +622,20 @@ mysql_generate_package_metadata() {
 # do a sanity check on the mysql database
 
 mysql_sanity_check() {
+  {
+    printf 'SELECT `binary_packages`.`pkgname`'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `build_assignments` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+    printf ' JOIN `architectures` as `bp_arch` ON `binary_packages`.`architecture`=`bp_arch`.`id`'
+    printf ' JOIN `architectures` as `ba_arch` ON `build_assignments`.`architecture`=`ba_arch`.`id`'
+    printf ' WHERE `bp_arch`.`name`!="any"'
+    printf ' AND `ba_arch`.`name`="any"'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '
+      1d
+      s/^/"any"-build assignment building non-"any" binary package: /
+    '
   ( # new shell is intentional
     temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir)
     trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
@@ -712,6 +726,35 @@ mysql_find_build_assignment_loops() {
 # clean up left overs from mysql database
 mysql_cleanup() {
   {
+    # remove dependencies w/o binary_package or install_target
+    printf 'DELETE FROM `dependencies` '
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `binary_packages` '
+      printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`'
+    printf ') ORNOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `install_targets` '
+      printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`'
+    printf ');\n'
+    # remove install_target_providers w/o binary_package or install_target
+    printf 'DELETE FROM `install_target_providers` '
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `binary_packages` '
+      printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`'
+    printf ') ORNOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `install_targets` '
+      printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`'
+    printf ');\n'
+    # remove build_assignments w/o binary_package
+    printf 'DELETE FROM `build_assignments` '
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `binary_packages` '
+      printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+    printf ');\n'
     # remove failed_builds with unbroken build_assignments
     printf 'DELETE `failed_builds` '
     printf 'FROM `failed_builds` '
@@ -725,6 +768,13 @@ mysql_cleanup() {
       printf 'SELECT * FROM `build_assignments` '
       printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`'
     printf ');\n'
+    # remove package_sources w/o build_assignment
+    printf 'DELETE FROM `package_sources` '
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `build_assignments` '
+      printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`'
+    printf ');\n'
   } | \
     ${mysql_command}
 }
-- 
cgit v1.2.3-54-g00ecf


From 765f9b249eed27f2b441107de8e2ebcbcbf874b4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 16:36:00 +0100
Subject: bin/get-package-updates: create metadata for _new_ build list
 packages

---
 bin/get-package-updates | 41 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 4fcefd9..a98c9ee 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -355,11 +355,48 @@ echo 'Extract dependencies of packages.'
 mkdir -p "${work_dir}/package-infos"
 
 while read -r package git_revision mod_git_revision repository; do
-
   generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
 done < "${work_dir}/build-list.new"
 
+{
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT DISTINCT '
+    printf 'replace(to_base64(`package_sources`.`%s`),"\\n",""),' \
+      'pkgbase' 'git_revision' 'mod_git_revision'
+    printf 'replace(to_base64(`upstream_repositories`.`name`),"\\n","")'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'repositories'          'repositories'          'binary_packages'   'repository' \
+      'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
+      'package_sources'       'package_sources'       'build_assignments' 'package_source' \
+      'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+    printf ' WHERE `repositories`.`name` in ("build-list","deletion-list")'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '
+      1d
+    ' | \
+    while read -r a b c d; do
+      printf '%s ' \
+        "$(printf '%s' "${a}" | base64 -d)" \
+        "$(printf '%s' "${b}" | base64 -d)" \
+        "$(printf '%s' "${c}" | base64 -d)" \
+        "$(printf '%s' "${d}" | base64 -d)" | \
+        sed 's/ /\n/'
+    done | \
+    sed '
+      y/\t/ /
+      p
+    '
+  cat "${work_dir}/build-list.new"
+} | \
+  sort | \
+  uniq -u | \
+  while read -r package git_revision mod_git_revision repository; do
+    mysql_generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
+  done
+
 echo 'apply blacklisting'
 # ignore blacklisted packages and dependent packages
 # this is the first time when all the information is available and up to date
-- 
cgit v1.2.3-54-g00ecf


From 6626de14465e6501035a0d5fe642e1cd308151bd Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 16:51:23 +0100
Subject: bin/mysql-functions: add quick-repair (removes falsely scheduled
 packages, but transfers all dependencies,installs to new packages)

---
 bin/mysql-functions | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index ea3ea78..c57b5ae 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -896,3 +896,39 @@ mysql_repair_binary_packages_without_build_assignment() {
     done | \
     ${mysql_command}
 }
+
+# mysql_remove_duplicate_build_order
+# remove _all_ build orders which just increase sub_pkgrel of another package
+# (this may undo "seed-build-list" effects)
+mysql_remove_duplicate_build_order() {
+  {
+    printf 'CREATE TEMPORARY TABLE `ren` (`old` BIGINT, `new` BIGINT);\n'
+    printf 'INSERT INTO `ren` (`old`,`new`)'
+    printf ' SELECT `old`.`id`,`new`.`id`'
+    printf ' FROM `binary_packages` as `old`'
+    printf ' JOIN `binary_packages` as `new` ON'
+    printf ' `old`.`%s`=`new`.`%s` AND' \
+      'pkgname' 'pkgname' \
+      'epoch' 'epoch' \
+      'pkgver' 'pkgver' \
+      'pkgrel' 'pkgrel'
+    printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`'
+    printf ' JOIN `repositories` AS `%s` ON `%s`.`id`=`%s`.`repository`' \
+      'orep' 'orep' 'old' \
+      'nrep' 'nrep' 'new'
+    printf ' WHERE `orep`.`name`!="build-list"'
+    printf ' AND `nrep`.`name`="build-list";\n'
+    printf 'UPDATE IGNORE `dependencies`'
+    printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`'
+    printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n'
+    printf 'UPDATE IGNORE `install_target_providers`'
+    printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`'
+    printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n'
+    printf 'DELETE FROM `binary_packages`'
+    printf ' WHERE EXISTS ('
+      printf 'SELECT * FROM `ren`'
+      printf ' WHERE `ren`.`old`=`binary_packages`.`id`'
+    printf ');\n'
+  } | \
+    ${mysql_command}
+}
-- 
cgit v1.2.3-54-g00ecf


From 2f06112371038694b6cac19e7619c8261e7baf1e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 17:04:19 +0100
Subject: bin/get-package-updates: reorder pipe

---
 bin/get-package-updates | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index a98c9ee..cfabace 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -376,6 +376,7 @@ done < "${work_dir}/build-list.new"
     ${mysql_command} --raw --batch | \
     sed '
       1d
+      y/\t/ /
     ' | \
     while read -r a b c d; do
       printf '%s ' \
@@ -385,10 +386,8 @@ done < "${work_dir}/build-list.new"
         "$(printf '%s' "${d}" | base64 -d)" | \
         sed 's/ /\n/'
     done | \
-    sed '
-      y/\t/ /
-      p
-    '
+    grep -vxF '' | \
+    sed 'p'
   cat "${work_dir}/build-list.new"
 } | \
   sort | \
-- 
cgit v1.2.3-54-g00ecf


From 610d38fb5510f0a415003701850d19d073ab162c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Feb 2018 23:29:04 +0100
Subject: bin/mysql-functions: even more tests

---
 bin/mysql-functions | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index c57b5ae..64f1915 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -623,18 +623,28 @@ mysql_generate_package_metadata() {
 
 mysql_sanity_check() {
   {
-    printf 'SELECT `binary_packages`.`pkgname`'
+    printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)'
     printf ' FROM `binary_packages`'
     printf ' JOIN `build_assignments` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
     printf ' JOIN `architectures` as `bp_arch` ON `binary_packages`.`architecture`=`bp_arch`.`id`'
     printf ' JOIN `architectures` as `ba_arch` ON `build_assignments`.`architecture`=`ba_arch`.`id`'
     printf ' WHERE `bp_arch`.`name`!="any"'
-    printf ' AND `ba_arch`.`name`="any"'
+    printf ' AND `ba_arch`.`name`="any";\n'
+    printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)'
+    printf ' FROM `binary_packages` AS `a`'
+    printf ' JOIN `binary_packages` AS `b`'
+    printf ' ON `a`.`pkgname`=`b`.`pkgname`'
+    printf ' AND `a`.`repository`=`b`.`repository`'
+    printf ' AND `a`.`id`!=`b`.`id`'
+    printf ' JOIN `repositories`'
+    printf ' ON `a`.`repository`=`repositories`.`id`'
+    printf ' WHERE `repositories`.`name`="build-list";\n'
   } | \
     ${mysql_command} --raw --batch | \
     sed '
-      1d
-      s/^/"any"-build assignment building non-"any" binary package: /
+      /^CONCAT("/d
+      s,^,<font color="#FF0000">,
+      s,$,</font>,
     '
   ( # new shell is intentional
     temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir)
-- 
cgit v1.2.3-54-g00ecf


From d1281c9b6f591201878aa092fdb4a2234cf8d31f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Feb 2018 14:05:11 +0100
Subject: bin/get-package-updates: bugfix black listing

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index cfabace..a8dc710 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -461,8 +461,8 @@ while [ -n "${black_listed_new}" ]; do
             printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`'
             printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`'
             printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
-            printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
           printf ')'
+          printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
         printf ');\n'
       } | \
         ${mysql_command} --raw --batch | \
-- 
cgit v1.2.3-54-g00ecf


From 98e89a079be0423449481606c4fc63bb9f9087c7 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Feb 2018 14:18:19 +0100
Subject: bin/mysql-functions: mysql_generate_package_metadata has more
 obligatory parameters now

---
 bin/bootstrap-mysql     |  2 +-
 bin/get-package-updates | 40 +---------------------------------------
 bin/mysql-functions     | 39 +++++++++++++++++++--------------------
 bin/seed-build-list     |  2 +-
 4 files changed, 22 insertions(+), 61 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 5acc2e2..1a50496 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -475,7 +475,7 @@ find "${work_dir}/package-states" \
 tr ' ' '.' < \
   "${work_dir}/build-list" | \
   while read -r state_file; do
-    mysql_generate_package_metadata "${state_file}"
+    mysql_generate_package_metadata 'build-list' "${state_file}"
   done
 
 # shellcheck disable=SC2016
diff --git a/bin/get-package-updates b/bin/get-package-updates
index a8dc710..8322e6d 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -337,7 +337,7 @@ done | \
           )"
       } | \
         ${mysql_command}
-      mysql_generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
+      mysql_generate_package_metadata 'build-list' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       continue
     fi
 
@@ -358,44 +358,6 @@ while read -r package git_revision mod_git_revision repository; do
   generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
 done < "${work_dir}/build-list.new"
 
-{
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT DISTINCT '
-    printf 'replace(to_base64(`package_sources`.`%s`),"\\n",""),' \
-      'pkgbase' 'git_revision' 'mod_git_revision'
-    printf 'replace(to_base64(`upstream_repositories`.`name`),"\\n","")'
-    printf ' FROM `binary_packages`'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'repositories'          'repositories'          'binary_packages'   'repository' \
-      'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
-      'package_sources'       'package_sources'       'build_assignments' 'package_source' \
-      'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
-    printf ' WHERE `repositories`.`name` in ("build-list","deletion-list")'
-  } | \
-    ${mysql_command} --raw --batch | \
-    sed '
-      1d
-      y/\t/ /
-    ' | \
-    while read -r a b c d; do
-      printf '%s ' \
-        "$(printf '%s' "${a}" | base64 -d)" \
-        "$(printf '%s' "${b}" | base64 -d)" \
-        "$(printf '%s' "${c}" | base64 -d)" \
-        "$(printf '%s' "${d}" | base64 -d)" | \
-        sed 's/ /\n/'
-    done | \
-    grep -vxF '' | \
-    sed 'p'
-  cat "${work_dir}/build-list.new"
-} | \
-  sort | \
-  uniq -u | \
-  while read -r package git_revision mod_git_revision repository; do
-    mysql_generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-  done
-
 echo 'apply blacklisting'
 # ignore blacklisted packages and dependent packages
 # this is the first time when all the information is available and up to date
diff --git a/bin/mysql-functions b/bin/mysql-functions
index 64f1915..19fb773 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -190,15 +190,16 @@ mysql_add_install_target() {
     ${mysql_command}
 }
 
-# mysql_generate_package_metadata $package $git_revision $mod_git_revision $repository
+# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository
 # or
-# mysql_generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# if sub_pkgrel should be determined automatically and the package is on the build-list
+# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository
+# if sub_pkgrel should be determined automatically
 # and
 # mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository
 # or
 # mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository
-# if $sub_pkgrel should be forced and the package is currently in $current_repository
+# if $sub_pkgrel should be forced
+
 # generate the meta data of a package (dependencies, built packages, ...) in the database
 
 mysql_generate_package_metadata() {
@@ -207,10 +208,6 @@ mysql_generate_package_metadata() {
     case "$1" in
       ''|*[!0-9]*)
         unset forced_sub_pkgrel
-        current_repository=$(
-          printf 'build-list' | \
-            base64 -w0
-        )
       ;;
       *)
         forced_sub_pkgrel=$(
@@ -218,21 +215,12 @@ mysql_generate_package_metadata() {
             base64 -w0
         )
         shift
-        current_repository=$(
-          printf '%s' "$1" | \
-            base64 -w0
-        )
-        shift
       ;;
     esac
-    package="$1"
-    git_revision="$2"
-    mod_git_revision="$3"
-    repository="$4"
-    temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
+    current_repository="$1"
+    package="$2"
 
-    if [ $# -eq 1 ]; then
+    if [ $# -eq 2 ]; then
       # second form
       repository="${package##*.}"
       package="${package%.*}"
@@ -240,8 +228,15 @@ mysql_generate_package_metadata() {
       package="${package%.*}"
       git_revision="${package##*.}"
       package="${package%.*}"
+    else
+      git_revision="$3"
+      mod_git_revision="$4"
+      repository="$5"
     fi
 
+    temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir)
+    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
+
     printf '.' >&2
     if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then
       printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"
@@ -276,6 +271,10 @@ mysql_generate_package_metadata() {
     printf '.' >&2
 
     # now we encode everything in base64
+    current_repository=$(
+      printf '%s' "${current_repository}" | \
+        base64 -w0
+    )
     pkgbase=$(
       printf '%s' "${pkgbase}" | \
         base64 -w0
diff --git a/bin/seed-build-list b/bin/seed-build-list
index e17242c..b049560 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -305,7 +305,7 @@ if ${update}; then
     tee -a "${work_dir}/build-list"
 
   while read -r git_revision mod_git_revision repository pkgbase; do
-    mysql_generate_package_metadata "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
+    mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
   done < \
     "${tmp_dir}/build-list.new"
 
-- 
cgit v1.2.3-54-g00ecf


From 11a53a1b63ec7b72e20c1e53ddfb613a712336c9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Feb 2018 14:21:28 +0100
Subject: bin/mysql-functions: fix typos in mysql_cleanup

---
 bin/mysql-functions | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 19fb773..f5a0f4b 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -741,7 +741,7 @@ mysql_cleanup() {
     printf '('
       printf 'SELECT * FROM `binary_packages` '
       printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`'
-    printf ') ORNOT EXISTS '
+    printf ') OR NOT EXISTS '
     printf '('
       printf 'SELECT * FROM `install_targets` '
       printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`'
@@ -752,7 +752,7 @@ mysql_cleanup() {
     printf '('
       printf 'SELECT * FROM `binary_packages` '
       printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`'
-    printf ') ORNOT EXISTS '
+    printf ') OR NOT EXISTS '
     printf '('
       printf 'SELECT * FROM `install_targets` '
       printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`'
-- 
cgit v1.2.3-54-g00ecf


From 5e3a4034972f57ddbc3738104541a81290298b09 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Feb 2018 14:23:16 +0100
Subject: bin/mysql-functions: mysql_remove_duplicate_build_order: work on
 to-be-decided instead of build-list

---
 bin/mysql-functions | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index f5a0f4b..cdc5f38 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -906,14 +906,15 @@ mysql_repair_binary_packages_without_build_assignment() {
     ${mysql_command}
 }
 
-# mysql_remove_duplicate_build_order
-# remove _all_ build orders which just increase sub_pkgrel of another package
-# (this may undo "seed-build-list" effects)
+# mysql_remove_duplicate_binary_packages
+# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel,
+# having difference of 1 in sub_pkgrel
+
 mysql_remove_duplicate_build_order() {
   {
-    printf 'CREATE TEMPORARY TABLE `ren` (`old` BIGINT, `new` BIGINT);\n'
-    printf 'INSERT INTO `ren` (`old`,`new`)'
-    printf ' SELECT `old`.`id`,`new`.`id`'
+    printf 'CREATE TEMPORARY TABLE `ren` (`old` BIGINT, `new` BIGINT, `repo` BIGINT);\n'
+    printf 'INSERT INTO `ren` (`old`,`new`,`repo`)'
+    printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`'
     printf ' FROM `binary_packages` as `old`'
     printf ' JOIN `binary_packages` as `new` ON'
     printf ' `old`.`%s`=`new`.`%s` AND' \
@@ -925,8 +926,8 @@ mysql_remove_duplicate_build_order() {
     printf ' JOIN `repositories` AS `%s` ON `%s`.`id`=`%s`.`repository`' \
       'orep' 'orep' 'old' \
       'nrep' 'nrep' 'new'
-    printf ' WHERE `orep`.`name`!="build-list"'
-    printf ' AND `nrep`.`name`="build-list";\n'
+    printf ' WHERE `orep`.`name`!="to-be-decided"'
+    printf ' AND `nrep`.`name`="to-be-decided";\n'
     printf 'UPDATE IGNORE `dependencies`'
     printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`'
     printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n'
@@ -938,6 +939,10 @@ mysql_remove_duplicate_build_order() {
       printf 'SELECT * FROM `ren`'
       printf ' WHERE `ren`.`old`=`binary_packages`.`id`'
     printf ');\n'
+    printf 'UPDATE IGNORE `binary_packages`'
+    printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`'
+    printf ' SET `binary_packages`.`repository`=`ren`.`repo`,'
+    printf ' `binary_packages`.`sub_pkgrel`=`binary_packages`.`sub_pkgrel`-1;\n'
   } | \
     ${mysql_command}
 }
-- 
cgit v1.2.3-54-g00ecf


From b66e5381ea809a44773a52aeb76f11a04e359271 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Feb 2018 21:26:56 +0100
Subject: bin/build-master-status-from-mysql: check locked packages, too

---
 bin/build-master-status-from-mysql | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 16afb9f..d1ee9ff 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -38,6 +38,31 @@ fi
 sort "${work_dir}/build-list" > \
   "${tmp_dir}/build-list.file"
 
+# shellcheck disable=SC2016
+{
+  printf 'SELECT '
+  printf '`package_sources`.`%s`,' \
+    'pkgbase' 'git_revision' 'mod_git_revision'
+  printf '`upstream_repositories`.`name`'
+  printf ' FROM `build_slaves`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \
+    'package_sources' 'package_sources' 'build_assignments' 'package_source' \
+    'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+} | \
+  ${mysql_command} --raw --batch | \
+  sed '
+    1d
+    y/\t/ /
+  ' | \
+  sort > \
+  "${tmp_dir}/locked.mysql"
+find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \
+  sed '
+    s/\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$/ \1 \2 \3/
+  ' | \
+  sort > \
+  "${tmp_dir}/locked.file"
 
 # shellcheck disable=SC2016
 {
@@ -193,6 +218,10 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   "${tmp_dir}/build-dependencies.file"
 
 {
+  diff -u \
+    "${tmp_dir}/locked.file" \
+    "${tmp_dir}/locked.mysql" || \
+    true
   mysql_sanity_check || true
   diff -u \
     "${tmp_dir}/build-list.file" \
-- 
cgit v1.2.3-54-g00ecf


From d67478ccbef5f540b597303120b2fd47317680cd Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Feb 2018 21:29:30 +0100
Subject: bin/mysql-functions: mysql_remove_duplicate_build_order: use
 sub_pkgrel from old package

---
 bin/mysql-functions | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index cdc5f38..16c0141 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -912,9 +912,10 @@ mysql_repair_binary_packages_without_build_assignment() {
 
 mysql_remove_duplicate_build_order() {
   {
-    printf 'CREATE TEMPORARY TABLE `ren` (`old` BIGINT, `new` BIGINT, `repo` BIGINT);\n'
-    printf 'INSERT INTO `ren` (`old`,`new`,`repo`)'
-    printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`'
+    printf 'CREATE TEMPORARY TABLE `ren`'
+    printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n'
+    printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)'
+    printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`'
     printf ' FROM `binary_packages` as `old`'
     printf ' JOIN `binary_packages` as `new` ON'
     printf ' `old`.`%s`=`new`.`%s` AND' \
@@ -942,7 +943,7 @@ mysql_remove_duplicate_build_order() {
     printf 'UPDATE IGNORE `binary_packages`'
     printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`'
     printf ' SET `binary_packages`.`repository`=`ren`.`repo`,'
-    printf ' `binary_packages`.`sub_pkgrel`=`binary_packages`.`sub_pkgrel`-1;\n'
+    printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n'
   } | \
     ${mysql_command}
 }
-- 
cgit v1.2.3-54-g00ecf


From cb6e59fd1e31842425b06722d99b57774fa4e49c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Feb 2018 21:38:37 +0100
Subject: bin/build-master-status-from-mysql: bugfix in adding of sub_pkgrel

---
 bin/build-master-status-from-mysql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index d1ee9ff..1d0b761 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -137,7 +137,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   -name '*.tested' \
   -exec cat '{}' \; | \
   sed '
-    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0-\2/
+    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/
     s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
   ' | \
   sort -u > \
@@ -173,7 +173,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   -name '*.testing' \
   -exec cat '{}' \; | \
   sed '
-    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0-\2/
+    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/
     s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
   ' | \
   sort -u > \
-- 
cgit v1.2.3-54-g00ecf


From 8a47d8320876599924e73a75a33c8fe3d20dc163 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 11 Feb 2018 14:00:34 +0100
Subject: bin/db-update: fix mysql expression - tables should have distinct
 names

---
 bin/db-update | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 9662ec4..f5ec07a 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -329,15 +329,19 @@ move_packages() {
   while read -r package; do
     while read -r part; do
       printf 'UPDATE `binary_packages`'
-      printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+      printf ' JOIN `repositories` AS `from_repo` ON `binary_packages`.`repository`=`from_repo`.`id`'
       printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
-      printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s"))' \
-        "$(
-          base64_encode_each < \
-            "${tmp_dir}/tmp/${package}.to_repo"
-        )"
+      printf ' SET `binary_packages`.`repository`=('
+        printf 'SELECT `to_repo`.`id`'
+        printf ' FROM `repositories` as `to_repo`'
+        printf ' WHERE `to_repo`.`name`=from_base64("%s")' \
+          "$(
+            base64_encode_each < \
+              "${tmp_dir}/tmp/${package}.to_repo"
+          )"
+        printf ')'
       printf ' WHERE'
-      printf ' `repositories`.`name`=from_base64("%s")' \
+      printf ' `from_repo`.`name`=from_base64("%s")' \
         "$(
           base64_encode_each < \
             "${tmp_dir}/tmp/${package}.from_repo"
-- 
cgit v1.2.3-54-g00ecf


From b8c6ba933adb5f4bad3ac28864a61d7e3b7166c1 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 11 Feb 2018 14:10:09 +0100
Subject: bin/get-package-updates: fix black listing mysql query

---
 bin/get-package-updates | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 8322e6d..612294e 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -403,9 +403,11 @@ while [ -n "${black_listed_new}" ]; do
       {
         printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
         printf 'INSERT INTO `bl` (`pkgbase`) VALUES '
-        printf '(from_base64("%s")),\n' "${black_listed}" | \
+        printf '%s\n' "${black_listed}" | \
           sort -u | \
           sed '
+            s/^/(from_base64("/
+            s/$/")),/
             $ s/,$/;/
           '
         printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")'
-- 
cgit v1.2.3-54-g00ecf


From 1ed82cccb9c46a715f033e1f83a7681b2d819d90 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 11 Feb 2018 14:39:46 +0100
Subject: bin/get-package-updates: add todo

---
 bin/get-package-updates | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 612294e..bddcf12 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -16,6 +16,8 @@
 # TODO: correctly handle if pkgbase of a split package is renamed, e.g.:
 # $a -> ($a,$b)  ==>  $b -> ($a,$b)
 
+# TODO: keep database clean in case of abort
+
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
-- 
cgit v1.2.3-54-g00ecf


From 9252ad4a7f3dbabf9b6c0e59219d94f3b1066a25 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 11 Feb 2018 20:45:42 +0100
Subject: bin/return-assignment: check expected packages from database

---
 bin/return-assignment | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 01a20c8..9dd7a2b 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -373,16 +373,33 @@ package_errors=$(
     # shellcheck disable=SC2086
     printf '%s\n' ${packages} | \
       sed '
-        s@\(-[^-]\+\)\{2\}-\([^-]\+\)\.pkg\.tar\.xz$@ \2@
-        / any$/{
-          s|any$|i686|
-        }
         s|^|was_built: |
       '
-    sed '
-      s|$| i686|
-      s|^|expected: |
-    ' "${work_dir}/package-infos/$1.$2.$3.$4.packages"
+    # shellcheck disable=SC2016
+    {
+      printf 'SELECT CONCAT('
+        printf '"expected: ",'
+        printf '`binary_packages`.`pkgname`,"-",'
+        printf 'if(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
+        printf '`binary_packages`.`pkgver`,"-",'
+        printf '`binary_packages`.`pkgrel`,".",'
+        printf '`binary_packages`.`sub_pkgrel`,"-",'
+        printf '`architectures`.`name`,".pkg.tar.xz"'
+      printf ')'
+      printf ' FROM `binary_packages`'
+      printf ' JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`'
+      printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`'
+      printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+        "$(
+          printf '%s' "${slave}" | \
+            base64 -w0
+        )"
+      printf ';\n'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '
+        1d
+      '
   } | \
     sort -k2 | \
     uniq -u -f1
-- 
cgit v1.2.3-54-g00ecf


From 3a10fa0e0834764c6a87f7ae93081361fc99d10e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 11 Feb 2018 21:05:26 +0100
Subject: bin/return-assignment: rely on db for checking if sent package is
 actually to be built currently

---
 bin/return-assignment | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 9dd7a2b..5560bd0 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -265,19 +265,35 @@ clean_up_lock_file() {
   rm -f "${package_database_lock_file}"
 }
 
-if ! grep -qxF "$1 $2 $3 $4" "${work_dir}/build-list" ||
-  ! [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ] ||
-  ! [ "$5" = "$(next_sub_pkgrel "$1" "$2" "$3" "$4")" ]; then
+# shellcheck disable=SC2016
+if ! {
+    printf 'SELECT count(*)'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+      'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
+      'build_slaves'          'build_assignments'     'build_slaves'      'currently_building' \
+      'repositories'          'repositories'          'binary_packages'   'repository' \
+      'package_sources'       'package_sources'       'build_assignments' 'package_source' \
+      'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_slaves`.`name`=from_base64("%s")' \
+      "$(printf '%s' "${slave}" | base64 -w0)"
+    printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+      'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
+      'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
+      'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+    printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+      "$(printf '%s' "$4" | base64 -w0)"
+    printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \
+      "$(printf '%s' "$5" | base64 -w0)"
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '1d' | \
+    grep -vxF '0'; then
   >&2 echo 'Sorry, the sent package is outdated.'
   exit 2
 fi
 
-# shellcheck disable=SC2154
-if ! grep -qxF "${slave}" "${work_dir}/package-states/$1.$2.$3.$4.locked"; then
-  >&2 echo 'Whoops, this package is not built by this slave.'
-  exit 2
-fi
-
 clean_up_tmp_dir() {
   cd "${base_dir}"
   rm -rf --one-file-system "${tmp_dir}"
-- 
cgit v1.2.3-54-g00ecf


From 5f7a55ba4f3145533010b17320aa14a584813724 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 11 Feb 2018 21:09:40 +0100
Subject: bin/get-package-updates: create metadata (in files) less thorough

---
 bin/get-package-updates | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index bddcf12..d752106 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -340,6 +340,7 @@ done | \
       } | \
         ${mysql_command}
       mysql_generate_package_metadata 'build-list' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
+      generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       continue
     fi
 
-- 
cgit v1.2.3-54-g00ecf


From c4a7f05c1eb634c5e2e6319c626c943ffb22f06f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 11 Feb 2018 21:26:29 +0100
Subject: bin/db-update: add todo

---
 bin/db-update | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index f5ec07a..13d5f32 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -22,6 +22,9 @@
 
 # TODO: read information from database
 
+# TODO: check if package is to be replaced in target repository
+# - this needs some special care in the database.
+
 # shellcheck disable=SC2039
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
-- 
cgit v1.2.3-54-g00ecf


From e2f5e61ff6a66bbf49af4d6f07f87eae6b4c3474 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Feb 2018 06:24:09 +0100
Subject: bin/mysql-functions: mysql_sanity_check: add check for differing
 sub_pkgrels on the build-list

---
 bin/mysql-functions | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 16c0141..262112e 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -638,6 +638,15 @@ mysql_sanity_check() {
     printf ' JOIN `repositories`'
     printf ' ON `a`.`repository`=`repositories`.`id`'
     printf ' WHERE `repositories`.`name`="build-list";\n'
+    printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)'
+    printf ' FROM `binary_packages` AS `a`'
+    printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`'
+    printf ' JOIN `repositories` AS `arep` ON `a`.`repository`=`arep`.`id`'
+    printf ' JOIN `repositories` AS `brep` ON `b`.`repository`=`brep`.`id`'
+    printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`'
+    printf ' AND `%srep`.`name` = "build-list"' \
+      'a' 'b'
+    printf ';\n'
   } | \
     ${mysql_command} --raw --batch | \
     sed '
-- 
cgit v1.2.3-54-g00ecf


From 1bee9d0041a1772d1066481d49bb349fa4adc28d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Feb 2018 13:46:07 +0100
Subject: bin/mysql-functions: add check if a single build assignment having
 packages on the build-list, deletion-list or to-beconsidered also has
 packages in a second "repository"

---
 bin/mysql-functions | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 262112e..ca78846 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -644,9 +644,25 @@ mysql_sanity_check() {
     printf ' JOIN `repositories` AS `arep` ON `a`.`repository`=`arep`.`id`'
     printf ' JOIN `repositories` AS `brep` ON `b`.`repository`=`brep`.`id`'
     printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`'
-    printf ' AND `%srep`.`name` = "build-list"' \
+    printf ' AND `%srep`.`name`="build-list"' \
       'a' 'b'
     printf ';\n'
+    printf 'SELECT DISTINCT CONCAT("package source ",'
+    printf '`package_sources`.`%s`,".",' \
+      'pkgbase' 'git_revision' 'mod_git_revision'
+    printf '`upstream_repositories`.`name`," has parts in ",'
+    printf '`repo_a`.`name`," and ",'
+    printf '`repo_b`.`name`," - which should not be the case!")'
+    printf ' FROM `upstream_repositories`'
+    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+      'package_sources'            'package_sources'   'upstream_package_repository' 'upstream_repositories' \
+      'build_assignments'          'build_assignments' 'package_source'              'package_sources' \
+      'binary_packages` AS `bin_a' 'bin_a'             'build_assignment'            'build_assignments' \
+      'binary_packages` AS `bin_b' 'bin_b'             'build_assignment'            'build_assignments' \
+      'repositories` AS `repo_a'   'bin_a'             'repository'                  'repo_a' \
+      'repositories` AS `repo_b'   'bin_b'             'repository'                  'repo_b'
+    printf ' WHERE `repo_a`.`name` IN ("build-list","deletion-list","to-be-decided")'
+    printf ' AND `bin_a`.`repository`!=`bin_b`.`repository`;\n'
   } | \
     ${mysql_command} --raw --batch | \
     sed '
-- 
cgit v1.2.3-54-g00ecf


From 1378f7609b70ed44926ed7a19822fe7c630e20be Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Feb 2018 14:25:39 +0100
Subject: bin/mysql-functions: add TODO

---
 bin/mysql-functions | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index ca78846..4b76d64 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -809,6 +809,7 @@ mysql_cleanup() {
       printf 'SELECT * FROM `build_assignments` '
       printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`'
     printf ');\n'
+ # TODO: remove handed out build-assignments which are not on the build-list
   } | \
     ${mysql_command}
 }
-- 
cgit v1.2.3-54-g00ecf


From a117834a6497ca868caa3943dd7275ad532e9d5a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Feb 2018 22:02:43 +0100
Subject: bin/mysql-functions: remove handed out build-assignments which are
 not on the build-list

---
 bin/mysql-functions | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 4b76d64..491b794 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -809,7 +809,11 @@ mysql_cleanup() {
       printf 'SELECT * FROM `build_assignments` '
       printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`'
     printf ');\n'
- # TODO: remove handed out build-assignments which are not on the build-list
+    printf 'UPDATE `build_slaves`'
+    printf ' JOIN `binary_packages` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`'
+    printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`'
+    printf ' SET `currently_building`=NULL'
+    printf ' WHERE `repositories`.`name`!="build-list";\n'
   } | \
     ${mysql_command}
 }
-- 
cgit v1.2.3-54-g00ecf


From f8d86b4fc47530ddb67819b00572a758d0c72789 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Feb 2018 22:14:54 +0100
Subject: bin/modify-package-state: mark all parts of a split package as
 "tested" (for now)

---
 bin/modify-package-state | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 0703238..b2d3133 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -230,7 +230,21 @@ while read -r package reason; do
     ${mysql_command} --raw --batch | \
     sed '1d' | \
     grep -qxF '1'; then
-    printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" | \
+    # shellcheck disable=SC2016
+    {
+      printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}"
+      printf ';\n'
+      # TODO: remove the below part, once the database if fully functional
+      printf 'UPDATE `binary_packages` AS `a`'
+      printf ' JOIN `binary_packages` AS `b`'
+      printf ' ON `a`.`build_assignment`=`b`.`build_assignment`'
+      printf ' JOIN `repositories`'
+      printf ' ON `b`.`repository`=`repositories`.`id`'
+      printf ' SET `b`.`is_tested`=1'
+      printf ' WHERE `a`.`is_tested`'
+      printf ' AND NOT `b`.`is_tested`'
+      printf ' AND NOT `repositories`.`name`="deletion-list";\n'
+    } | \
       ${mysql_command}
     case "${action}" in
       'block')
-- 
cgit v1.2.3-54-g00ecf


From d1bb013a5072bf9c9ef4846331e7331e3667b943 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Feb 2018 22:31:49 +0100
Subject: bin/get-package-updates: delay update of build-list in database

---
 bin/get-package-updates | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index d752106..07e88a2 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -185,6 +185,17 @@ if ! flock -s ${block_flag} 8; then
   exit
 fi
 
+# shellcheck disable=SC2016
+{
+  printf 'DELETE FROM `binary_packages`'
+  printf ' WHERE `binary_packages`.`repository`=('
+    printf 'SELECT `repositories`.`id`'
+    printf ' FROM `repositories`'
+    printf ' WHERE `repositories`.`name`="to-be-decided"'
+  printf ')'
+} | \
+  ${mysql_command}
+
 echo 'Check modified packages from the last update, and put them to the build list.'
 
 # Check modified packages from the last update, and put them to the build list.
@@ -573,6 +584,20 @@ else
   rm "${work_dir}/tsort.error"
 fi
 
+# shellcheck disable=SC2016
+{
+  printf 'UPDATE `binary_packages`'
+  printf ' JOIN `repositories` AS `from_repo`'
+  printf ' ON `binary_packages`.`repository`=`from_repo`.`id`'
+  printf ' SET `repository`=('
+    printf 'SELECT `to_repo`.`id`'
+    printf ' FROM `repositories` AS `to_repo`'
+    printf ' WHERE `to_repo`.`name`="build-list"'
+  printf ')'
+  printf ' WHERE `from_repo`.`name`="to-be-decided";\n'
+} | \
+  ${mysql_command}
+
 # update loop list in database
 mysql_find_build_assignment_loops
 
-- 
cgit v1.2.3-54-g00ecf


From fd342d4b92aa6881df2903804829a54100b83367 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Feb 2018 22:53:40 +0100
Subject: bin/get-package-updates: actually put new packages into
 "to-be-decided" instead of "build-list"

---
 bin/get-package-updates | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 07e88a2..533f14f 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -106,7 +106,7 @@ delete_package() {
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
   # shellcheck disable=SC2016
   {
-    # packages from the build-list go straight to the deletion-list
+    # packages from the build-list/to-be-decided go straight to the deletion-list
     printf 'UPDATE `binary_packages`'
     printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
       'repositories' 'repositories' 'repository' \
@@ -114,7 +114,7 @@ delete_package() {
     printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
       'package_sources' 'package_sources' 'build_assignments' 'package_source'
     printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
-    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")'
     printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \
       "$(printf '%s' "$1" | base64 -w0)"
     # other packages are marked as `is_to_be_deleted`
@@ -350,7 +350,7 @@ done | \
           )"
       } | \
         ${mysql_command}
-      mysql_generate_package_metadata 'build-list' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
+      mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       continue
     fi
-- 
cgit v1.2.3-54-g00ecf


From 85e1b4ef9f0da06b6bcb2a3c8e550fdd1bb8b794 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 10:38:17 +0100
Subject: bin/get-package-updates: test_exclusion now cooperates with the
 database

---
 bin/get-package-updates | 57 ++++++++++++++++++++++++++-----------------------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 533f14f..465f896 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -9,8 +9,6 @@
 
 # TODO: Find out, why sometimes package updates are missed.
 
-# TODO: test_exclusion does not yet cooperate with the database
-
 # TODO: read information from database
 
 # TODO: correctly handle if pkgbase of a split package is renamed, e.g.:
@@ -104,31 +102,36 @@ delete_package() {
   echo "$1" >> \
     "${work_dir}/deletion-list.new"
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
-  # shellcheck disable=SC2016
-  {
-    # packages from the build-list/to-be-decided go straight to the deletion-list
-    printf 'UPDATE `binary_packages`'
-    printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
-      'repositories' 'repositories' 'repository' \
-      'build_assignments' 'build_assignments' 'build_assignment'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'package_sources' 'package_sources' 'build_assignments' 'package_source'
-    printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
-    printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")'
-    printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \
-      "$(printf '%s' "$1" | base64 -w0)"
-    # other packages are marked as `is_to_be_deleted`
-    printf 'UPDATE `binary_packages`'
-    printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
-      'repositories' 'repositories' 'repository' \
-      'build_assignments' 'build_assignments' 'build_assignment'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'package_sources' 'package_sources' 'build_assignments' 'package_source'
-    printf ' SET `binary_packages`.`is_to_be_deleted`=1'
-    printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \
-      "$(printf '%s' "$1" | base64 -w0)"
-  } | \
-    ${mysql_command}
+  # TODO: Once we want to rely on the database for test_exclusion, we
+  # need to run the command below unconditionally, but with some
+  # changes, so we can easily revert.
+  if [ -z "${test_exclusion}" ]; then
+    # shellcheck disable=SC2016
+    {
+      # packages from the build-list/to-be-decided go straight to the deletion-list
+      printf 'UPDATE `binary_packages`'
+      printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
+        'repositories' 'repositories' 'repository' \
+        'build_assignments' 'build_assignments' 'build_assignment'
+      printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+        'package_sources' 'package_sources' 'build_assignments' 'package_source'
+      printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
+      printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")'
+      printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \
+        "$(printf '%s' "$1" | base64 -w0)"
+      # other packages are marked as `is_to_be_deleted`
+      printf 'UPDATE `binary_packages`'
+      printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
+        'repositories' 'repositories' 'repository' \
+        'build_assignments' 'build_assignments' 'build_assignment'
+      printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+        'package_sources' 'package_sources' 'build_assignments' 'package_source'
+      printf ' SET `binary_packages`.`is_to_be_deleted`=1'
+      printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \
+        "$(printf '%s' "$1" | base64 -w0)"
+    } | \
+      ${mysql_command}
+  fi
 }
 
 # create tmp_dir and trap
-- 
cgit v1.2.3-54-g00ecf


From a9f0449be9a12ffd6842671eee49fde40b3196f3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 10:48:25 +0100
Subject: bin/get-package-updates: cleanup db in trap on exit

---
 bin/get-package-updates | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 465f896..abd380e 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -135,8 +135,20 @@ delete_package() {
 }
 
 # create tmp_dir and trap
+clean_up() {
+  rm -rf --one-file-system "${tmp_dir}"
+  # shellcheck disable=SC2016
+  {
+    printf 'DELETE FROM `binary_packages`'
+    printf ' WHERE `binary_packages`.`repository`=('
+      printf 'SELECT `repositories`.`id` FROM `repositories`'
+      printf ' WHERE `repositories`.`name`="to-be-decided"'
+    printf ');\n'
+  } | \
+    ${mysql_command}
+}
 tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+trap 'clean_up' EXIT
 
 # Update git repositories (official packages, community packages and the repository of package customizations).
 
-- 
cgit v1.2.3-54-g00ecf


From 07ed3c88e045d84ab282272c9f1b675c15487ba1 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 11:20:08 +0100
Subject: bin/sanity-check: check full name of package in database

---
 bin/sanity-check | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/bin/sanity-check b/bin/sanity-check
index e31a5ba..39a2d10 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -270,17 +270,18 @@ while [ $# -gt 0 ]; do
 
         errors=$(
           {
-            tar -tzf "${tmp_dir}/${repo}.db.tar.gz" | \
-              grep '/$' | \
-              sed '
-                s|/$||
-                s|^|in_database |
+            tar -Oxzf "${tmp_dir}/${repo}.db.tar.gz" --wildcards '*/desc' 2>/dev/null | \
+              sed -n '
+                /^%FILENAME%$/ {
+                  N
+                  s/^.*\n/in_database /
+                  p
+                }
               '
             echo "${packages}" | \
-              grep '\S' | \
               sed '
-                s|-[^-]\+$||
-                s|^|in_repository |
+                /\.pkg\.tar\.xz$/ !d
+                s/^/in_repository /
               ' | \
               sort -u
           } | \
-- 
cgit v1.2.3-54-g00ecf


From c4ead6b43d4731065939e38ee7d8977ce50158ce Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 11:49:23 +0100
Subject: bin/mysql-functions: do not match the sub_pkgrel on inserting
 provided install_targets and dependencies

---
 bin/mysql-functions | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 491b794..68bc37d 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -583,8 +583,12 @@ mysql_generate_package_metadata() {
               'pkgver' "${pkgver}" \
               'pkgrel' "${pkgrel}" \
               'pkgname' "${pkgname}"
-            printf ' `binary_packages`.`sub_pkgrel` = %s AND' \
-              "${sub_pkgrel}"
+            # we do not want to match the sub_pkgrel:
+            # a) it is tedious to do so (because it may be calculated
+            #    dynamically)
+            # b) it is not necessary to do so: if only the sub_pkgrel
+            #    changed, the dependencies and provided install_targets
+            #    should not have changed
             printf ' `architectures`.`name` = from_base64("%s") AND' \
               "${arch}"
             printf ' `repositories`.`name` = from_base64("%s");\n' \
-- 
cgit v1.2.3-54-g00ecf


From ecf59e38604bf4c299387019606a74ceb20495ac Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 11:57:11 +0100
Subject: bin/mysql-functions: remove a sanity check which should not be an
 issue

---
 bin/mysql-functions | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 68bc37d..62ced74 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -651,22 +651,6 @@ mysql_sanity_check() {
     printf ' AND `%srep`.`name`="build-list"' \
       'a' 'b'
     printf ';\n'
-    printf 'SELECT DISTINCT CONCAT("package source ",'
-    printf '`package_sources`.`%s`,".",' \
-      'pkgbase' 'git_revision' 'mod_git_revision'
-    printf '`upstream_repositories`.`name`," has parts in ",'
-    printf '`repo_a`.`name`," and ",'
-    printf '`repo_b`.`name`," - which should not be the case!")'
-    printf ' FROM `upstream_repositories`'
-    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-      'package_sources'            'package_sources'   'upstream_package_repository' 'upstream_repositories' \
-      'build_assignments'          'build_assignments' 'package_source'              'package_sources' \
-      'binary_packages` AS `bin_a' 'bin_a'             'build_assignment'            'build_assignments' \
-      'binary_packages` AS `bin_b' 'bin_b'             'build_assignment'            'build_assignments' \
-      'repositories` AS `repo_a'   'bin_a'             'repository'                  'repo_a' \
-      'repositories` AS `repo_b'   'bin_b'             'repository'                  'repo_b'
-    printf ' WHERE `repo_a`.`name` IN ("build-list","deletion-list","to-be-decided")'
-    printf ' AND `bin_a`.`repository`!=`bin_b`.`repository`;\n'
   } | \
     ${mysql_command} --raw --batch | \
     sed '
-- 
cgit v1.2.3-54-g00ecf


From 8d1294a3ba224c0a1b998772329fd6b4563ec6e6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 13:26:45 +0100
Subject: bin/return-assignment: remove done TODO

---
 bin/return-assignment | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 5560bd0..a1f3020 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -16,8 +16,6 @@
 
 # TODO: sign database
 
-# TODO: read information from database
-
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-- 
cgit v1.2.3-54-g00ecf


From 1c172d7c63a5594c33d4f9408857fe56eddf2694 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 13:29:48 +0100
Subject: bin/db-update: remove done TODO

---
 bin/db-update | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 13d5f32..83b3370 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -16,10 +16,6 @@
 
 # TODO: handle dependencies of parts of a split package separately
 
-# TODO: improve tracking of packages:
-# track packages in stable, too
-# know _exactly_ in which repository the packages are
-
 # TODO: read information from database
 
 # TODO: check if package is to be replaced in target repository
-- 
cgit v1.2.3-54-g00ecf


From ab55e44e1917556c453850add1fd1bb32ad0bb1f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 13:30:54 +0100
Subject: bin/db-update: remove unneccessary TODO

---
 bin/db-update | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 83b3370..e5ea193 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -18,9 +18,6 @@
 
 # TODO: read information from database
 
-# TODO: check if package is to be replaced in target repository
-# - this needs some special care in the database.
-
 # shellcheck disable=SC2039
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
-- 
cgit v1.2.3-54-g00ecf


From 5b9639e6ef09230933ec6a572130336bcef57feb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 13:52:49 +0100
Subject: bin/mysql-functions: mysql_cleanup now has a dry-run capability

---
 bin/mysql-functions | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 62ced74..241c84b 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -744,12 +744,23 @@ mysql_find_build_assignment_loops() {
     ${mysql_command}
 }
 
-# mysql_cleanup
+# mysql_cleanup [dry]
 # clean up left overs from mysql database
 mysql_cleanup() {
+  local operator
+  if [ "$#" = '0' ]; then
+    operator='DELETE'
+  elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then
+    operator='SELECT COUNT(*)'
+  else
+    >&2 echo 'Unknown parameter'
+    >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".'
+    return 1
+  fi
   {
     # remove dependencies w/o binary_package or install_target
-    printf 'DELETE FROM `dependencies` '
+    printf '%s FROM `dependencies` ' \
+      "${operator}"
     printf 'WHERE NOT EXISTS '
     printf '('
       printf 'SELECT * FROM `binary_packages` '
@@ -760,7 +771,8 @@ mysql_cleanup() {
       printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`'
     printf ');\n'
     # remove install_target_providers w/o binary_package or install_target
-    printf 'DELETE FROM `install_target_providers` '
+    printf '%s FROM `install_target_providers` ' \
+      "${operator}"
     printf 'WHERE NOT EXISTS '
     printf '('
       printf 'SELECT * FROM `binary_packages` '
@@ -771,27 +783,30 @@ mysql_cleanup() {
       printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`'
     printf ');\n'
     # remove build_assignments w/o binary_package
-    printf 'DELETE FROM `build_assignments` '
+    printf '%s FROM `build_assignments` ' \
+      "${operator}"
     printf 'WHERE NOT EXISTS '
     printf '('
       printf 'SELECT * FROM `binary_packages` '
       printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`'
     printf ');\n'
     # remove failed_builds with unbroken build_assignments
-    printf 'DELETE `failed_builds` '
-    printf 'FROM `failed_builds` '
+    printf '%s FROM `failed_builds` ' \
+      "${operator}"
     printf 'JOIN `build_assignments` '
     printf 'ON `failed_builds`.`build_assignment`=`build_assignments`.`id` '
     printf 'WHERE NOT `build_assignments`.`is_broken`;\n'
     # remove failed_builds w/o build_assignment
-    printf 'DELETE FROM `failed_builds` '
+    printf '%s FROM `failed_builds` ' \
+      "${operator}"
     printf 'WHERE NOT EXISTS '
     printf '('
       printf 'SELECT * FROM `build_assignments` '
       printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`'
     printf ');\n'
     # remove package_sources w/o build_assignment
-    printf 'DELETE FROM `package_sources` '
+    printf '%s FROM `package_sources` ' \
+      "${operator}"
     printf 'WHERE NOT EXISTS '
     printf '('
       printf 'SELECT * FROM `build_assignments` '
-- 
cgit v1.2.3-54-g00ecf


From 2782ca1872ca1276df85f5a039418d1f472e103a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 14:17:10 +0100
Subject: bin/return-assignment: only move packages from the build-list which
 are on the build-list

---
 bin/return-assignment | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index a1f3020..7d8a609 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -481,21 +481,23 @@ fi
   printf 'UPDATE `build_assignments`'
   printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
   printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+  printf ' JOIN `repositories` AS `old_repo` ON `binary_packages`.`repository`=`old_repo`.`id`'
   printf ' SET'
   printf ' `build_assignments`.`is_broken`=0,'
   printf ' `build_assignments`.`priority`=0,'
-  printf ' `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s")),' \
+  printf ' `binary_packages`.`repository`=(SELECT `new_repo`.`id` FROM `repositories` AS `new_repo` WHERE `new_repo`.`name`=from_base64("%s")),' \
     "$(
       printf '%s' "${destination}" | \
         base64 -w0
     )"
   printf ' `binary_packages`.`has_issues`=0,'
   printf ' `binary_packages`.`is_tested`=0'
-  printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
+  printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
     "$(
       printf '%s' "${slave}" | \
         base64 -w0
     )"
+  printf ' AND `old_repo`.`name`="build-list";\n'
   printf 'UPDATE `build_slaves` AS `to_update`'
   printf ' JOIN `build_slaves` AS `current_slave`'
   printf ' ON `to_update`.`currently_building`=`current_slave`.`currently_building`'
-- 
cgit v1.2.3-54-g00ecf


From 5b40c2766152357f448e1c425ae999d2f3d244d7 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 15:21:31 +0100
Subject: bin/mysql-functions: bugfix in mysql_cleanup

---
 bin/mysql-functions | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 241c84b..1cd68cf 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -793,9 +793,11 @@ mysql_cleanup() {
     # remove failed_builds with unbroken build_assignments
     printf '%s FROM `failed_builds` ' \
       "${operator}"
-    printf 'JOIN `build_assignments` '
-    printf 'ON `failed_builds`.`build_assignment`=`build_assignments`.`id` '
-    printf 'WHERE NOT `build_assignments`.`is_broken`;\n'
+    printf 'WHERE ('
+      printf 'SELECT NOT `build_assignments`.`is_broken`'
+      printf ' FROM `build_assignments` '
+      printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
+    printf ');\n'
     # remove failed_builds w/o build_assignment
     printf '%s FROM `failed_builds` ' \
       "${operator}"
-- 
cgit v1.2.3-54-g00ecf


From e3021bb9a8bcd9d58e69034312354f9a45dccd3f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 15:33:39 +0100
Subject: bin/build-master-status-from-mysql: ignore version of dependencies
 for now in the sanity_check

---
 bin/build-master-status-from-mysql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 1d0b761..6f5dea1 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -207,8 +207,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   "${tmp_dir}/build-dependencies.mysql"
 {
   cat "${tmp_dir}/build-dependencies.mysql"
+  # TODO: consider versions dependencies, too
   find "${work_dir}/package-infos" -name '*.build-depends' \
-    -exec grep -HF '' {} \; | \
+    -exec grep -Hv '[<=>]' {} \; | \
     sed '
       s,^.*/,,
       s/\.\([^.]\+\)\.\([^.]\+\)\.\([^.:]\+\)\.build-depends:\([^:]\+\)$/ \1 \2 \3 \4/
-- 
cgit v1.2.3-54-g00ecf


From 53235d7796b666cb678483fa12d5c527d9c7b854 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 15:46:02 +0100
Subject: bin/return-assignment: only build-list packages are expected to have
 been built

---
 bin/return-assignment | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/return-assignment b/bin/return-assignment
index 7d8a609..e107124 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -403,11 +403,13 @@ package_errors=$(
       printf ' FROM `binary_packages`'
       printf ' JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`'
       printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`'
+      printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
       printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
         "$(
           printf '%s' "${slave}" | \
             base64 -w0
         )"
+      printf ' AND `repositories`.`name`="build-list"'
       printf ';\n'
     } | \
       ${mysql_command} --raw --batch | \
-- 
cgit v1.2.3-54-g00ecf


From 98e596252854ad77272b264e8124259bb141c5fb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 18:28:39 +0100
Subject: bin/mysql-functions: repository should not matter for
 dependencies/provided

---
 bin/mysql-functions | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 1cd68cf..f30c9a2 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -567,8 +567,6 @@ mysql_generate_package_metadata() {
             printf ' `binary_packages`'
             printf ' JOIN `architectures`'
             printf ' ON `binary_packages`.`architecture` = `architectures`.`id`'
-            printf ' JOIN `repositories`'
-            printf ' ON `binary_packages`.`repository` = `repositories`.`id`'
             printf ' JOIN `install_targets`'
             printf ' JOIN `%s`' "${link}"
             printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
@@ -589,10 +587,10 @@ mysql_generate_package_metadata() {
             # b) it is not necessary to do so: if only the sub_pkgrel
             #    changed, the dependencies and provided install_targets
             #    should not have changed
-            printf ' `architectures`.`name` = from_base64("%s") AND' \
+            printf ' `architectures`.`name` = from_base64("%s");\n' \
               "${arch}"
-            printf ' `repositories`.`name` = from_base64("%s");\n' \
-              "${current_repository}"
+            # the repository is of no relevance: it hardly matters for
+            # the dependencies
           done
 
           printf 'DROP TABLE `%s`;\n' \
-- 
cgit v1.2.3-54-g00ecf


From 846f000605d39adb4bc4e72e4ebb92760c46be2a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 20:55:57 +0100
Subject: bin/build-master-status-from-mysql: only check dependencies of
 package sources which have binary packages currently (real or virtual)

---
 bin/build-master-status-from-mysql | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 6f5dea1..cd85156 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -207,13 +207,32 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   "${tmp_dir}/build-dependencies.mysql"
 {
   cat "${tmp_dir}/build-dependencies.mysql"
-  # TODO: consider versions dependencies, too
-  find "${work_dir}/package-infos" -name '*.build-depends' \
-    -exec grep -Hv '[<=>]' {} \; | \
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT DISTINCT '
+    printf '`package_sources`.`%s`,' \
+      'pkgbase' 'git_revision' 'mod_git_revision'
+    printf '`upstream_repositories`.`name`'
+    printf ' FROM `upstream_repositories`'
+    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+      'package_sources'   'package_sources'   'upstream_package_repository' 'upstream_repositories' \
+      'build_assignments' 'build_assignments' 'package_source'              'package_sources' \
+      'binary_packages'   'binary_packages'   'build_assignment'            'build_assignments'
+  } | \
+    ${mysql_command} --raw --batch | \
     sed '
-      s,^.*/,,
-      s/\.\([^.]\+\)\.\([^.]\+\)\.\([^.:]\+\)\.build-depends:\([^:]\+\)$/ \1 \2 \3 \4/
-    '
+      1d
+      y/\t/ /
+    ' | \
+    while read -r pkgbase git_revision mod_git_revision repository; do
+      if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then
+        # TODO: consider versions dependencies, too
+        grep -v '[<=>]' "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" | \
+          sed '
+            s/^/'"${pkgbase} ${git_revision} ${mod_git_revision} ${repository}"' /
+          '
+      fi
+    done
 } | \
   sort -u > \
   "${tmp_dir}/build-dependencies.file"
-- 
cgit v1.2.3-54-g00ecf


From aa008f2bf3b5016ed8eaa3946f7e3cee9f739ce9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Feb 2018 21:19:44 +0100
Subject: web-scripts/broken-packages.php: correctly link git revision of
 upstream

---
 web-scripts/broken-packages.php | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
index 3468681..4dabf5c 100644
--- a/web-scripts/broken-packages.php
+++ b/web-scripts/broken-packages.php
@@ -109,7 +109,10 @@ if ($result -> num_rows > 0) {
         $rows[$count]["git_revision"] =
           $rows[$count]["git_revision"] . "x86_64";
       $rows[$count]["git_revision"] =
-        $rows[$count]["git_revision"] . "/\">" .
+        $rows[$count]["git_revision"] . "?id=" .
+        $row["git_revision"];
+      $rows[$count]["git_revision"] =
+        $rows[$count]["git_revision"] . "\">" .
         $row["git_revision"] . "</a>";
     } else
       $rows[$count]["git_revision"] = $row["git_revision"];
-- 
cgit v1.2.3-54-g00ecf


From f822d014afa5a48a0335bd5babd97ecf137d9e76 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 11:25:25 +0100
Subject: bin/build-master-status-from-mysql: should take sanity-check lock (so
 it currently cannot be run in parallel with sanity-check)

---
 bin/build-master-status-from-mysql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index cd85156..7a6605a 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -11,8 +11,8 @@ tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
 # always block if locked
-exec 9> "${build_list_lock_file}"
-if ! flock -n -s 9; then
+exec 9> "${sanity_check_lock_file}"
+if ! flock -n 9; then
   >&2 echo 'Mysql-Sanity check skipped, cannot acquire lock.'
   exit
 fi
-- 
cgit v1.2.3-54-g00ecf


From 85b734363309052f5c5392751df574130c4d93a5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 11:45:17 +0100
Subject: bin/mysql-functions: do not increase sub_pkgrel for packages in
 $current_repo, too (-> split packages should receive identical sub_pkgrels!)

---
 bin/mysql-functions | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index f30c9a2..d1a8c2a 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -424,7 +424,9 @@ mysql_generate_package_metadata() {
         else
           sub_pkgrel=$(
             printf '(SELECT COALESCE('
-            # do not add binary packages which are currently on the build-list
+            # do not add binary packages which are currently on the
+            # build-list or in $current_repository (beware of split
+            # packages!)
             printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
             printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
             printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
@@ -436,7 +438,8 @@ mysql_generate_package_metadata() {
               'pkgname' "${pkgname}"
             printf ' `architectures`.`name`=from_base64("%s")' \
               "${arch}"
-            printf ' AND `repositories`.`name`="build-list"),'
+            printf ' AND `repositories`.`name` IN ("build-list",from_base64("%s"))),' \
+              "${current_repository}"
             # max(sub_pkgrel)+1
             printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
             printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
-- 
cgit v1.2.3-54-g00ecf


From 437fa0fc93e4c59b7d677a4d954dbd4656448b15 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 14:10:04 +0100
Subject: bin/mysql-functions: "exit 2" rather than "return 2" on syntactical
 error

---
 bin/mysql-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index d1a8c2a..2f0b282 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -756,7 +756,7 @@ mysql_cleanup() {
   else
     >&2 echo 'Unknown parameter'
     >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".'
-    return 1
+    exit 2
   fi
   {
     # remove dependencies w/o binary_package or install_target
-- 
cgit v1.2.3-54-g00ecf


From a844091fa0913b7b18b1f2a087f717085384a3c6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 14:11:49 +0100
Subject: bin/mysql-functions: mysql_package_name_query new

---
 bin/mysql-functions | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 2f0b282..78728c0 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -983,3 +983,14 @@ mysql_remove_duplicate_build_order() {
   } | \
     ${mysql_command}
 }
+
+mysql_package_name_query() {
+  printf 'CONCAT('
+  printf '`binary_packages`.`pkgname`,"-",'
+  printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
+  printf '`binary_packages`.`pkgver`,"-",'
+  printf '`binary_packages`.`pkgrel`,".",'
+  printf '`binary_packages`.`sub_pkgrel`,"-",'
+  printf '`architectures`.`name`,".pkg.tar.xz"'
+  printf ')'
+}
-- 
cgit v1.2.3-54-g00ecf


From b7b8706cf9221a880f79262d98d507f6e64c3d07 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 14:20:31 +0100
Subject: bin/get-package-updates: update git hashes in database, too

---
 bin/get-package-updates | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index abd380e..0b4d4f5 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -601,6 +601,15 @@ fi
 
 # shellcheck disable=SC2016
 {
+  # update hashes of repositories in mysql database
+  for repo in ${repo_names}; do
+    printf 'UPDATE `git_repositories`'
+    printf ' SET `git_repositories`.`head`=from_base64("%s")' \
+      "$(eval 'printf '"'"'%s'"'"' "${repo_revisions__'"${repo}"'}" | base64 -w0')"
+    printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
+      "$(printf '%s' "${repo}" | base64 -w0)"
+  done
+  # move binary_packages from "to-be-decided" to "build-list"
   printf 'UPDATE `binary_packages`'
   printf ' JOIN `repositories` AS `from_repo`'
   printf ' ON `binary_packages`.`repository`=`from_repo`.`id`'
-- 
cgit v1.2.3-54-g00ecf


From 777b3ed99dfa55d15d0b3012f4cf2d91d5df7184 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 15:35:10 +0100
Subject: bin/return-assignment: use mysql_package_name_query()

---
 bin/return-assignment | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index e107124..7b2dec3 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -393,12 +393,7 @@ package_errors=$(
     {
       printf 'SELECT CONCAT('
         printf '"expected: ",'
-        printf '`binary_packages`.`pkgname`,"-",'
-        printf 'if(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
-        printf '`binary_packages`.`pkgver`,"-",'
-        printf '`binary_packages`.`pkgrel`,".",'
-        printf '`binary_packages`.`sub_pkgrel`,"-",'
-        printf '`architectures`.`name`,".pkg.tar.xz"'
+        mysql_package_name_query
       printf ')'
       printf ' FROM `binary_packages`'
       printf ' JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`'
-- 
cgit v1.2.3-54-g00ecf


From fcb02682d01c6a39d9313897bc43dfe566188f15 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 15:43:37 +0100
Subject: bin/seed-build-list: use solely the database

---
 bin/seed-build-list | 403 ++++++++++++++++++++++++++--------------------------
 1 file changed, 199 insertions(+), 204 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index b049560..bfe975e 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -5,8 +5,6 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: read information from database
-
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
@@ -45,7 +43,6 @@ eval set -- "$(
   )"
 
 touch "${tmp_dir}/mirrors"
-touch "${tmp_dir}/delta-packages"
 touch "${tmp_dir}/package-regexes"
 touch "${tmp_dir}/ignore-packages"
 
@@ -99,220 +96,218 @@ fi
 
 repos="${stable_package_repositories}"
 
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f\n' | \
-  sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
-  sort -k1,1 > \
-  "${tmp_dir}/known-packages"
-
-mod_repo_rev=$(cat "${work_dir}/archlinux32.revision")
-{
-  for repo in ${repo_names}; do
-    if [ "${repo_names}" = 'archlinux32' ]; then
-      continue
-    fi
-    eval 'repo_path="${repo_paths__'"${repo}"'}"'
-    repo_rev=$(cat "${work_dir}/${repo}.revision")
-    git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
-      tar -t | \
-      grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
-      grep -v -- '-i686/PKGBUILD$' | \
-      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
+# genereate must-haves query from mirror delta
+if [ -s "${tmp_dir}/mirrors" ]; then
+  {
+    # theirs
+    while read -r mirror; do
+      if [ -z "${mirror}" ]; then
+        continue
+      fi
+      for repo in ${repos}; do
+        curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
+          tar -Oxz --wildcards '*/desc' | \
+          sed '
+            /^%FILENAME%$/!d
+            N
+            s/^.*\n//
+            s/-x86_64\(\.pkg\.tar\.xz\)$/-i686\1/
+            s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/theirs \2 \3 \1/
+          '
+      done
+    done < \
+      "${tmp_dir}/mirrors"
+    # ours
+    # shellcheck disable=SC2016
+    {
+      printf 'SELECT '
+      mysql_package_name_query
+      printf ' FROM `binary_packages`'
+      printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+    } | \
+      ${mysql_command} --raw --batch | \
       sed '
-        s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|'"${repo_rev}"' \1 \2|
+        1d
+        s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/
       '
-  done | \
-    sort -u | \
-    sort -k1,1
-  awk '{print "nothing " $1 " " $4}' "${tmp_dir}/known-packages" | \
-    sort -u | \
-    sed 'p'
-} | \
-  sort -k2,3 | \
-  uniq -uf1 | \
-  while read -r repo_rev pkg prepo; do
-    generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}"
-  done
-
-{
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT '
-    printf '`package_sources`.`%s`,' \
-      'pkgbase' 'git_revision' 'mod_git_revision'
-    printf '`upstream_repositories`.`name`,`binary_packages`.`pkgname`'
-    printf ' FROM `binary_packages`'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
-      'package_sources'       'package_sources'       'build_assignments' 'package_source' \
-      'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
   } | \
-    ${mysql_command} --raw --batch | \
-    sed '
-      1d
-      y/\t/ /
-    '
-  find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
-    sed '
-      s|^.*/||
-      s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 |
-    '
-} | \
-  sort -u | \
-  sort -k5,5 > \
-  "${tmp_dir}/known-packages"
-
-# generate delta-packages from package-regexes
-while read -r pkg_regex; do
-  if [ -z "${pkg_regex}" ]; then
-    continue
-  fi
-  awk '{print $5}' "${tmp_dir}/known-packages" | \
-    grep "${pkg_regex}" || \
-    true
-done < \
-  "${tmp_dir}/package-regexes" >> \
-  "${tmp_dir}/delta-packages"
-
-# genereate delta_packages from mirror delta
+    expand_version 2 | \
+    sort -k3,4 -k2Vr,2 -k1,1 | \
+    shrink_version 2 | \
+    uniq -f2 | \
+    sed -n '
+      s/^theirs \(\S\+ \)\{2\}//
+      T
+      p
+    ' | \
+    sort -u > \
+    "${tmp_dir}/must-haves"
+fi
 
-while read -r mirror; do
-  if [ -z "${mirror}" ]; then
-    continue
-  fi
-  {
-    # theirs
-    for repo in ${repos}; do
-      curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
-        tar -tz
-    done | \
-      grep '/$' | \
+# shellcheck disable=SC2016
+{
+  printf 'CREATE TEMPORARY TABLE `ignore_packages` (`pkgname` VARCHAR(64));\n'
+  if [ -s "${tmp_dir}/ignore-packages" ]; then
+    grep -vxF '' "${tmp_dir}/ignore-packages" | \
+      base64_encode_each | \
       sed '
-        s|/$||
-        s|^\(.*\)-\([^-]\+-[^-]\+\)|theirs \1-\2 \2 \1|
+        s/^/(from_base64("/
+        s/$/")),/
+        $s/,$/;/
+        1 s/^/INSERT INTO `ignore_packages` (`pkgname`) VALUES \n/
       '
+  fi
+  # packages on the build-list or deletion-list should be ignored
+  printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)'
+  printf ' SELECT DISTINCT `binary_packages`.`pkgname`'
+  printf ' FROM `binary_packages`'
+  printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+  printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list");\n'
+} | \
+  sponge "${tmp_dir}/ignore-packages"
 
-    # ours
-    for repo in $(ls_master_mirror 'i686'); do
-      ls_master_mirror "i686/${repo}" | \
-        grep '\.pkg\.tar\.xz$' | \
-        sed 's|-[^-]\+$||'
-    done | \
+# shellcheck disable=SC2016
+{
+  printf 'CREATE TEMPORARY TABLE `must_haves` (`pkgname` VARCHAR(64));\n'
+  if [ -s "${tmp_dir}/must-haves" ]; then
+    grep -vxF '' "${tmp_dir}/must-haves" | \
+      base64_encode_each | \
       sed '
-        s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1|
+        s/^/(from_base64("/
+        s/$/")),/
+        $s/,$/;/
+        1 s/^/INSERT INTO `must_haves` (`pkgname`) VALUES \n/
       '
-  } | \
-    expand_version 3 | \
-    sort -k4,4 -k3Vr,3 -k1,1 | \
-    shrink_version 3 | \
-    uniq -f3 | \
-    grep '^theirs ' | \
-    awk '{print $4}' | \
-    sort -k1,1 >> \
-    "${tmp_dir}/delta-packages"
-done < \
-  "${tmp_dir}/mirrors"
-
-sort -k1,1 -u "${tmp_dir}/delta-packages" | \
-  sponge "${tmp_dir}/delta-packages"
-
-cat \
-  "${tmp_dir}/delta-packages" \
-  "${tmp_dir}/ignore-packages" \
-  "${tmp_dir}/ignore-packages" | \
-  sort | \
-  uniq -u | \
-  sponge "${tmp_dir}/delta-packages"
-
-join -1 1 -2 5 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/delta-packages" "${tmp_dir}/known-packages" >> \
-  "${tmp_dir}/append-packages"
+  fi
+} | \
+  sponge "${tmp_dir}/must-haves"
 
+# fetch unknown must-haves from upstream
+# shellcheck disable=SC2016
+printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` VARCHAR(64));\n' > \
+  "${tmp_dir}/pkgbases"
+# shellcheck disable=SC2016
 {
-  awk '{print $5}' "${tmp_dir}/append-packages"
-  cat "${tmp_dir}/delta-packages"
+  cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages"
+  printf 'SELECT `must_haves`.`pkgname` FROM `must_haves`'
+  printf ' WHERE NOT EXISTS ('
+    printf 'SELECT * FROM `binary_packages`'
+    printf ' WHERE `binary_packages`.`pkgname`=`must_haves`.`pkgname`'
+  printf ') AND NOT EXISTS ('
+    printf 'SELECT * FROM `ignore_packages`'
+    printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`'
+  printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n'
 } | \
-  sort | \
-  uniq -u | \
-  sponge "${tmp_dir}/delta-packages"
-
-if [ -s "${tmp_dir}/delta-packages" ]; then
-  echo 'There are still packages newer for x86_64 which I cannot identify :-/'
-  cat "${tmp_dir}/delta-packages"
-  exit 2
-fi
-
-rm "${tmp_dir}/delta-packages"
-
-# convert append-packages to build-list.new (add git hashes)
-
-cut -d' ' -f1,4 < \
-  "${tmp_dir}/append-packages" | \
-  sort -u | \
-  while read -r pkg repo; do
-    git_repo=$(find_git_repository_to_package_repository "${repo}")
-    printf '%s %s %s %s\n' \
-      "${pkg}" \
-      "$(cat "${work_dir}/${git_repo}.revision")" \
-      "$(cat "${work_dir}/archlinux32.revision")" \
-      "${repo}"
+  ${mysql_command} --raw --batch | \
+  sed '1d' | \
+  while read -r pkgname; do
+    content=$(
+      curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
+        tr ',' '\n'
+    )
+    repo=$(
+      printf '%s\n' "${content}" | \
+        sed -n '
+          s/^\s*"repo"\s*:\s*"//
+          T
+          s/".*$//
+          T
+          p
+        '
+    )
+    pkgbase=$(
+      printf '%s\n' "${content}" | \
+        sed -n '
+          s/^\s*"pkgbase"\s*:\s*"//
+          T
+          s/".*$//
+          T
+          p
+        '
+    )
+    if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
+      >&2 printf 'Could not find "%s" which is newer on x86_64!?\n' "${pkgname}"
+      exit 2
+    fi
+    printf '(from_base64("%s"),from_base64("%s")),\n' \
+      "$(printf '%s' "${pkgbase}" | base64 -w0)" \
+      "$(printf '%s' "${repo}" | base64 -w0)"
   done | \
-  sort -u > \
-  "${tmp_dir}/build-list.append"
-
-# Create a lock file for build list.
-
-if ${update}; then
-  # always block if locked
-  exec 9> "${build_list_lock_file}"
-  flock 9
-
-  exec 8> "${sanity_check_lock_file}"
-  flock -s 8
-fi
+  sort -u | \
+  sed '
+    1 s/^/INSERT IGNORE INTO `pkgbases` (`pkgbase`,`repository`) VALUES \n/
+    $s/,$/;/
+  ' >> \
+  "${tmp_dir}/pkgbases"
 
+# shellcheck disable=SC2016
 {
-  awk '{print $2 " " $3 " " $4 " " $1}' "${tmp_dir}/build-list.append"
-
-  # ignore packages on the build-list
-  awk '{print $2 " " $3 " " $4 " " $1}' < \
-    "${work_dir}/build-list" | \
-    sed 'p'
-
-  # ignore packages on the deletion-list
-  awk '{print "X X X " $1}' "${work_dir}/deletion-list" | \
-    sed 'p'
-
-  # ignore packages on the blacklist
-  git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
-    tar -Ox blacklist | \
-    sed '
-      s/\s*#.*$//
-      /^\s*$/d
-    ' | \
-    awk '{print "X X X " $1}' | \
-    sed 'p'
-
-  # ignore explicitely ignored packages
-  awk '{print "X X X " $1}' "${tmp_dir}/ignore-packages" | \
-    sed 'p'
+  cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages" "${tmp_dir}/pkgbases"
+  printf 'SELECT '
+  printf '`pkgbases`.`pkgbase`,'
+  printf '`git_repositories`.`head`,'
+  printf '('
+    printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`'
+    printf ' WHERE `al32`.`name`="archlinux32"'
+  printf '),'
+  printf '`pkgbases`.`repository`'
+  printf ' FROM `pkgbases`'
+  printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`'
+  printf ' JOIN `git_repositories` ON `git_repositories`.`id`=`upstream_repositories`.`git_repository`'
+  printf ';\n'
+  printf 'SELECT '
+  printf '`package_sources`.`pkgbase`,'
+  printf '`git_repositories`.`head`,'
+  printf '('
+    printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`'
+    printf ' WHERE `al32`.`name`="archlinux32"'
+  printf '),'
+  printf '`upstream_repositories`.`name`'
+  printf ' FROM `binary_packages`'
+  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
+    'build_assignments'     'build_assignments'     'binary_packages'       'build_assignment' \
+    'package_sources'       'package_sources'       'build_assignments'     'package_source' \
+    'upstream_repositories' 'upstream_repositories' 'package_sources'       'upstream_package_repository' \
+    'git_repositories'      'git_repositories'      'upstream_repositories' 'git_repository'
+  printf ' WHERE ('
+  if [ -s "${tmp_dir}/package-regexes" ]; then
+    grep -vxF '' "${tmp_dir}/package-regexes" | \
+      base64_encode_each | \
+      sed '
+        s/^/`binary_packages`.`pkgname` REGEXP from_base64("/
+        s/$/") OR /
+      ' | \
+      tr -d '\n'
+  fi
+  printf 'EXISTS ('
+    printf 'SELECT * FROM `must_haves`'
+    printf ' WHERE `must_haves`.`pkgname`=`binary_packages`.`pkgname`'
+  printf ')) AND NOT EXISTS ('
+    printf 'SELECT * FROM `ignore_packages`'
+    printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`'
+  printf ');\n'
 } | \
-  sort -k4,4 | \
-  uniq -uf3 > \
-  "${tmp_dir}/build-list.new"
-
-if ${update}; then
-  awk '{print $4 " " $1 " " $2 " " $3}' \
-    "${tmp_dir}/build-list.new" | \
-    tee -a "${work_dir}/build-list"
-
-  while read -r git_revision mod_git_revision repository pkgbase; do
-    mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
-  done < \
-    "${tmp_dir}/build-list.new"
-
-  # Remove the lock file
-
-  rm -f "${build_list_lock_file}"
-else
-  awk '{print $4 " " $1 " " $2 " " $3}' \
-    "${tmp_dir}/build-list.new"
-fi
+  ${mysql_command} --raw --batch | \
+  sort -u | \
+  sed '
+    y/\t/ /
+    / [0-9a-f]\{40\} [0-9a-f]\{40\} /!d
+  ' | \
+  if ${update}; then
+    # always block if locked
+    exec 9> "${build_list_lock_file}"
+    flock 9
+
+    exec 8> "${sanity_check_lock_file}"
+    flock -s 8
+
+    while read -r pkgbase git_rev mod_git_rev repo; do
+      printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
+      mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
+      printf '\n' >&2
+    done
+
+    rm -f "${build_list_lock_file}"
+  else
+    cat
+  fi
-- 
cgit v1.2.3-54-g00ecf


From b4eb9d2dcda28b0107b0bde28f4f4426bb08af05 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Feb 2018 23:48:30 +0100
Subject: bin/get-assignment: do not use next_sub_pkgrel, but take info
 straight from database

---
 bin/common-functions | 64 ----------------------------------------------------
 bin/get-assignment   | 33 ++++++++++++++++++++++-----
 2 files changed, 27 insertions(+), 70 deletions(-)

diff --git a/bin/common-functions b/bin/common-functions
index 3a03d27..bc43686 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -1237,70 +1237,6 @@ print_list_of_archaic_packages() {
     sort -u
 }
 
-# next_sub_pkgrel $package $git_revision $mod_git_revision $repository
-# giv out the next sub-pkgrel of the given package
-
-next_sub_pkgrel() {
-  ( # new shell is intentional
-    package="$1"
-    git_revision="$2"
-    mod_git_revision="$3"
-    repository="$4"
-    git_repo=$(find_git_repository_to_package_repository "${repository}")
-
-    temp_dir=$(mktemp -d 'tmp.common-functions.next_sub_pkgrel.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
-    find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
-    extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${temp_dir}" '0'
-
-    version=$(
-      cd "${temp_dir}"
-      makepkg --printsrcinfo | \
-        sed -n '
-          /^\s*\(epoch\|pkg\(name\|ver\|rel\)\) = /{s|^\s\+||;p}
-        ' | \
-      sed '
-        /^pkgname = /{
-          s/^.*= //
-          w pkgnames
-          d
-        }
-        s|^epoch = \(.*\)$|1 \1:|
-        s|^pkgver = \(.*\)$|2 \1-|
-        s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1|
-      ' | \
-        sort -k1n,1 | \
-        sed '
-          s|^[0-9] ||
-          :a
-          N
-          s|\n[0-9] \(\S\+\)$|\1|
-        ta
-      '
-    )
-    sub_pkgrel=$(
-      ls_master_mirror 'i686/*/' | \
-        sed -n "$(
-          sed '
-            s/$/-'"${version}"'/
-            s/\./\\./g
-            s/^/^/
-            s/$/\\(\\.\\([0-9]\\+\\)\\)\\?-[^-]\\+$/
-            s|^.*$|/\0/{ s/\0/\\2/; s/^$/0/; p; b; }|
-          ' "${temp_dir}/pkgnames"
-        )" | \
-        sort -n | \
-        tail -n1
-    )
-    if [ -z "${sub_pkgrel}" ]; then
-      echo '0'
-    else
-      echo "$((sub_pkgrel+1))"
-    fi
-  )
-}
-
 # modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
 # print the given modification revision possibly with a html link to github
 
diff --git a/bin/get-assignment b/bin/get-assignment
index 67c7aa4..a7eec2d 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -22,11 +22,6 @@ mkdir -p "${work_dir}/package-states"
 
 hand_out_assignment() {
 
-  # find out the sub_pkgrel
-  sub_pkgrel=$(
-    next_sub_pkgrel "$1" "$2" "$3" "$4"
-  )
-
   # we don't care anymore if an older version of this package was
   # "locked" or "broken"
   find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
@@ -43,7 +38,33 @@ hand_out_assignment() {
     $ a '"$1 $2 $3 $4" \
     "${work_dir}/build-list"
 
-  echo "$1 $2 $3 $4 ${sub_pkgrel}"
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT '
+    printf '`package_sources`.`%s`,' \
+      'pkgbase' 'git_revision' 'mod_git_revision'
+    printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`'
+    printf ' FROM `upstream_repositories`'
+    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
+      'package_sources'   'package_sources'   'upstream_package_repository' 'upstream_repositories' \
+      'build_assignments' 'build_assignments' 'package_source'              'package_sources' \
+      'binary_packages'   'binary_packages'   'build_assignment'            'build_assignments' \
+      'repositories'      'binary_packages'   'repository'                  'repositories'
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+      'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
+      'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
+      'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+    printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+      "$(printf '%s' "$4" | base64 -w0)"
+    printf ' LIMIT 1;\n'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '
+      1d
+      y/\t/ /
+    '
+
   {
     # shellcheck disable=SC2154
     echo "${slave}"
-- 
cgit v1.2.3-54-g00ecf


From a6fc54fea6be7bb1e5a64a8429d8832d8456b19f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 00:20:56 +0100
Subject: bin/get-package-updates: repair blacklisting

---
 bin/get-package-updates | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 0b4d4f5..67e3f73 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -456,6 +456,12 @@ while [ -n "${black_listed_new}" ]; do
             printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
           printf ')'
           printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+        printf ') AND EXISTS ('
+        # TODO: This should be corrected at the root: automatic install targets, which are bogus should
+        #       not be added in the first place - but how do we detect that?
+          printf ' SELECT *'
+          printf ' FROM `install_target_providers`'
+          printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
         printf ');\n'
       } | \
         ${mysql_command} --raw --batch | \
-- 
cgit v1.2.3-54-g00ecf


From e7b14b22c38ed83e5d0d8c473c1c65423e11c9bd Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 13:13:57 +0100
Subject: bin/bootstrap-mysql: create `repository_stability_relations`

---
 bin/bootstrap-mysql | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 1a50496..c965319 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -133,7 +133,10 @@ if [ ! "$1" = 'slim' ]; then
       ' description VARCHAR(512)'
       'todo_links MEDIUMINT' \
       ' dependent MEDIUMINT' \
-      ' depending_on MEDIUMINT'
+      ' depending_on MEDIUMINT' \
+      'repository_stability_relations MEDIUMINT' \
+      ' more_stable MEDIUMINT :repository_stabilities' \
+      ' less_stable MEDIUMINT :repository_stabilities'
   )
 
   {
@@ -396,6 +399,30 @@ fi
   } | \
     sed 's|,;|;|'
 
+  # shellcheck disable=SC2016
+  {
+    printf 'INSERT IGNORE INTO `repository_stability_relations` (`more_stable`,`less_stable`)'
+    printf ' SELECT `ms`.`id`,`ls`.`id`'
+    printf ' FROM `repository_stabilities` AS `ms` JOIN `repository_stabilities` AS `ls`'
+    printf ' WHERE '
+    printf '(`ms`.`name`="%s" AND `ls`.`name`="%s") OR ' \
+      'stable' 'stable' \
+      'stable' 'testing' \
+      'stable' 'staging' \
+      'stable' 'unbuilt' \
+      'stable' 'forbidden' \
+      'testing' 'testing' \
+      'testing' 'staging' \
+      'testing' 'unbuilt' \
+      'testing' 'forbidden' \
+      'staging' 'staging' \
+      'staging' 'unbuilt' \
+      'staging' 'forbidden' \
+      'unbuilt' 'forbidden'
+    printf ';\n'
+  } | \
+    sed 's| OR ;|;|'
+
   for repo in \
     'core:stable:AQ==' \
     'extra:stable:AQ==' \
-- 
cgit v1.2.3-54-g00ecf


From ab3d96872cdac3b814592b55ca3a0cb159d626be Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 13:17:05 +0100
Subject: bin/get-package-updates: remove safety net which calls
 "generate_package_metadata" on _all_ packages

---
 bin/get-package-updates | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 67e3f73..89b09df 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -377,16 +377,6 @@ done | \
 sort -u "${work_dir}/deletion-list.new" | \
   sponge "${work_dir}/deletion-list.new"
 
-echo 'Extract dependencies of packages.'
-
-# First, we extract the dependencies of each package.
-
-mkdir -p "${work_dir}/package-infos"
-
-while read -r package git_revision mod_git_revision repository; do
-  generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-done < "${work_dir}/build-list.new"
-
 echo 'apply blacklisting'
 # ignore blacklisted packages and dependent packages
 # this is the first time when all the information is available and up to date
-- 
cgit v1.2.3-54-g00ecf


From f82e8e45ebf97b9b30afe1e2742e2f540d652b39 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 14:11:18 +0100
Subject: bin/ii-connect: fix regex recognizing buildmaster commands in channel

---
 bin/ii-connect | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/ii-connect b/bin/ii-connect
index cfe5043..867381e 100755
--- a/bin/ii-connect
+++ b/bin/ii-connect
@@ -51,10 +51,10 @@ if [ "$1" = 'watch' ]; then
         else
           prefix=''
         fi
-        regex='^\S\+ \S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) '
+        regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) '
         if grep -q "${regex}" "${out_file}"; then
           sed -n '
-            s/'"${regex}"'/\1 /
+            s/'"${regex}"'/\2 /
             T
             p
           ' "${out_file}" | \
-- 
cgit v1.2.3-54-g00ecf


From cfb696f3ccbc5ae1aa934d467952b2a3087f31ab Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 14:35:46 +0100
Subject: bin/common-functions: remove_old_package_versions - start with db

---
 bin/common-functions | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/bin/common-functions b/bin/common-functions
index bc43686..4ac57c6 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -426,6 +426,91 @@ remove_old_package_versions() {
   local repository="$2"
   local package="$3"
 
+  local pkgname
+  local epoch
+  local pkgver
+  local pkgrel
+  local sub_pkgrel
+  local arch
+  pkgname="${package%.pkg.tar.xz}"
+  arch="${pkgname##*-}"
+  pkgname="${pkgname%-*}"
+  pkgrel="${pkgname##*-}"
+  sub_pkgrel="${pkgrel##*.}"
+  if [ "${sub_pkgrel}" = "${pkgrel}" ]; then
+    sub_pkgrel='0'
+  else
+    pkgrel="${pkgrel%.*}"
+  fi
+  pkgname="${pkgname%-*}"
+  pkgver="${pkgname##*-}"
+  epoch="${pkgver%%:*}"
+  if [ "${epoch}" = "${pkgver}" ]; then
+    epoch='0'
+  else
+    pkgver="${pkgver#*:}"
+  fi
+  pkgname="${pkgname%-*}"
+
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \
+      "$(
+        printf '%s' "${repository}" | \
+          base64 -w0
+      )" \
+      "$(
+        printf '%s' "${package}" | \
+          base64 -w0
+      )" \
+      "$(
+        printf '%s' "${package}" | \
+          sed '
+            s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/
+          ' | \
+          base64 -w0
+      )"
+    printf 'SELECT '
+    printf '`binary_packages`.`id`,'
+    printf 'CONCAT(`repositories`.`name`,"/",'
+    mysql_package_name_query
+    printf '),'
+    # should we delete packages of identical version?
+    printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),'
+    printf 'CONCAT('
+      printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
+      printf '`binary_packages`.`pkgver`,"-",'
+      printf '`binary_packages`.`pkgrel`,".",'
+      printf '`binary_packages`.`sub_pkgrel`'
+    printf ')'
+    printf ' FROM `binary_packages`'
+    printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
+      'repositories'  'repositories'  'repository' \
+      'architectures' 'architectures' 'architecture'
+    printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
+    printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`'
+    # name must match
+    printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \
+      "$(printf '%s' "${package%-*-*-*}" | base64 -w0)"
+    # repository, where package should be deleted, should be less stable
+    printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \
+      "$(printf '%s' "${repository}" | base64 -w0)"
+    printf ';\n'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '
+      /^\S\+\sCONCAT(/d
+      y/\t/ /
+    ' | \
+    expand_version 4 | \
+    sort -k4V,4 -k3r,3 | \
+    shrink_version 4 | \
+    sed -n '
+      /^bogus /q
+      p
+    ' | \
+    cut -d' ' -f1,2 >&2
+
   # repositories in which older packages should be deleted
   local delete_older_repositories
   # repositories in which not-newer packages should be deleted
-- 
cgit v1.2.3-54-g00ecf


From b89e738fb2d34d1efa48f20a8cb13644b371378e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 14:56:07 +0100
Subject: bin/why-dont-you: remove obsolete commented code

---
 bin/why-dont-you | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/why-dont-you b/bin/why-dont-you
index 83dc35f..e9922b7 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -114,8 +114,6 @@ case "${action}" in
         s/NULL,//g
       ' | \
       while read -r id is_broken trials is_blocked dependency slave pkgbase; do
-#      printf '"%s" ' "$id" "$is_broken" "$trials" "$is_blocked" "$dependency" "$slave" "$pkgbase" >&2
-#      printf '\n' >&2
         pkgbase=$(
           printf '%s' "${pkgbase}" | \
             base64 -d
-- 
cgit v1.2.3-54-g00ecf


From 33322fbda73861befb4b56299a23915b7d72bbe5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 14:57:35 +0100
Subject: bin/mysql-functions: separate checkdependencies from makedependencies

---
 bin/bootstrap-mysql |  2 +-
 bin/mysql-functions | 41 +++++++++++++++++++++++++++++++----------
 2 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index c965319..322e0a1 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -458,7 +458,7 @@ fi
   printf 'INSERT IGNORE INTO `dependency_types` (`name`,`relevant_for_building`,`relevant_for_binary_packages`) VALUES'
   {
     printf '\n  ("%s",%s,%s),' \
-      'build' '1' '0' \
+      'make' '1' '0' \
       'check' '0' '0' \
       'link' '0' '1' \
       'run' '1' '1'
diff --git a/bin/mysql-functions b/bin/mysql-functions
index 78728c0..0a8e3d3 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -407,8 +407,14 @@ mysql_generate_package_metadata() {
             sed 's/[<>=].*$//' | \
             base64_encode_each
         )
-        builddepends=$(
-          grep '^\(checkdepends\|makedepends\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+        makedepends=$(
+          grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+            cut -d' ' -f3 | \
+            sed 's/[<>=].*$//' | \
+            base64_encode_each
+        )
+        checkdepends=$(
+          grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
             cut -d' ' -f3 | \
             sed 's/[<>=].*$//' | \
             base64_encode_each
@@ -514,7 +520,8 @@ mysql_generate_package_metadata() {
         {
           printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \
             'provides' \
-            'builddepends' \
+            'makedepends' \
+            'checkdepends' \
             'rundepends'
 
           printf 'INSERT INTO `provides` VALUES\n'
@@ -538,8 +545,19 @@ mysql_generate_package_metadata() {
             '
           printf ' ("base");\n'
 
-          printf 'INSERT INTO `builddepends` VALUES\n'
-          echo "${builddepends}" | \
+          echo "${checkdepends}" | \
+            sort -u | \
+            grep -vxF '' | \
+            sed '
+              1 s/^/INSERT INTO `checkdepends` VALUES \n/
+              s|^| (from_base64("|
+              s|$|")),|
+              $ s/,$/;/
+            '
+          printf ' ("base-devel");\n'
+
+          printf 'INSERT INTO `makedepends` VALUES\n'
+          echo "${makedepends}" | \
             sort -u | \
             grep -vxF '' | \
             sed '
@@ -551,17 +569,18 @@ mysql_generate_package_metadata() {
           printf 'INSERT IGNORE INTO `install_targets` (`name`)'
           printf ' SELECT (`name`) FROM `%s` UNION' \
             'provides' \
-            'builddepends' \
+            'makedepends' \
+            'checkdepends' \
             'rundepends' | \
             sed 's| UNION$|;\n|'
 
-          for link in 'provides' 'builddepends' 'rundepends'; do
+          for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
             case "${link}" in
               'provides')
                 printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT'
                 printf ' `binary_packages`.`id`,`install_targets`.`id` FROM'
               ;;
-              'builddepends'|'rundepends')
+              'makedepends'|'checkdepends'|'rundepends')
                 printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT'
                 printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM'
                 printf ' `dependency_types` JOIN'
@@ -574,7 +593,8 @@ mysql_generate_package_metadata() {
             printf ' JOIN `%s`' "${link}"
             printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
             printf ' WHERE'
-            if [ "${link}" = 'builddepends' ] || \
+            if [ "${link}" = 'makedepends' ] || \
+              [ "${link}" = 'checkdepends' ] || \
               [ "${link}" = 'rundepends' ]; then
               printf ' `dependency_types`.`name` = "%s" AND' \
                 "${link%depends}"
@@ -598,7 +618,8 @@ mysql_generate_package_metadata() {
 
           printf 'DROP TABLE `%s`;\n' \
             'provides' \
-            'builddepends' \
+            'makedepends' \
+            'checkdepends' \
             'rundepends'
         } >> \
           "${temp_dir}/add-install-targets-command"
-- 
cgit v1.2.3-54-g00ecf


From 1c5d2f6a45acce10cc240508e6a99fd3ce66a29a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 15:11:36 +0100
Subject: bin/mysql-functions: whoops - remove left over line

---
 bin/mysql-functions | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 0a8e3d3..90179c3 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -554,7 +554,6 @@ mysql_generate_package_metadata() {
               s|$|")),|
               $ s/,$/;/
             '
-          printf ' ("base-devel");\n'
 
           printf 'INSERT INTO `makedepends` VALUES\n'
           echo "${makedepends}" | \
-- 
cgit v1.2.3-54-g00ecf


From 45ca53e3a30afb02d30cbcf1ef16fa40278731f0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 15:19:36 +0100
Subject: bin/common-functions: $arch should not be extracted from $pkgname in
 remove_old_package_versions

---
 bin/common-functions | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bin/common-functions b/bin/common-functions
index 4ac57c6..bcf8ed2 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -431,9 +431,6 @@ remove_old_package_versions() {
   local pkgver
   local pkgrel
   local sub_pkgrel
-  local arch
-  pkgname="${package%.pkg.tar.xz}"
-  arch="${pkgname##*-}"
   pkgname="${pkgname%-*}"
   pkgrel="${pkgname##*-}"
   sub_pkgrel="${pkgrel##*.}"
-- 
cgit v1.2.3-54-g00ecf


From 74b5ceb19b64d8f7f92320dfd269c5f6ceaf5bb2 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 15:49:41 +0100
Subject: bin/common-functions: fix typo - arrrgh

---
 bin/common-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/common-functions b/bin/common-functions
index bcf8ed2..d0786a9 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -431,7 +431,7 @@ remove_old_package_versions() {
   local pkgver
   local pkgrel
   local sub_pkgrel
-  pkgname="${pkgname%-*}"
+  pkgname="${package%-*}"
   pkgrel="${pkgname##*-}"
   sub_pkgrel="${pkgrel##*.}"
   if [ "${sub_pkgrel}" = "${pkgrel}" ]; then
-- 
cgit v1.2.3-54-g00ecf


From a62f25a873bc0dca01076848e72271bee6874162 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 15:52:51 +0100
Subject: bin/mysql-functions: mysql_generate_package_metadata: fix sedding for
 mysql

---
 bin/mysql-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 90179c3..7f2cdab 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -549,9 +549,9 @@ mysql_generate_package_metadata() {
             sort -u | \
             grep -vxF '' | \
             sed '
-              1 s/^/INSERT INTO `checkdepends` VALUES \n/
               s|^| (from_base64("|
               s|$|")),|
+              1 s/^/INSERT INTO `checkdepends` VALUES \n/
               $ s/,$/;/
             '
 
-- 
cgit v1.2.3-54-g00ecf


From f32d45f3baed4589033107d6463a4b9d11c877c6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Feb 2018 16:08:24 +0100
Subject: bin/common-functions: ignore checkdepends in the state-files for now
 - they're still in the database

---
 bin/common-functions | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/bin/common-functions b/bin/common-functions
index d0786a9..222062e 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -183,17 +183,14 @@ generate_package_metadata() {
     sort -u > \
     "${file_prefix}.builds"
 
-  # TODO: checkdepends should be _much_ less important!
-
-  # extract "build-depends" = makedepends \cup checkdepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
+  # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
   {
     {
       printf 'all_depend = %s\n' 'base' 'base-devel'
       sed -n "$(
         printf '/^\t%s = /p\n' \
           'depends' \
-          'makedepends' \
-          'checkdepends'
+          'makedepends'
       )" "${file_prefix}.SRCINFO"
     } | \
       cut -d= -f2 | \
-- 
cgit v1.2.3-54-g00ecf


From d7a7e5cebf55c13ab3522e6ec17e11cba878da73 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Feb 2018 09:00:54 +0100
Subject: bin/common-functions: fix typo in comment

---
 bin/common-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/common-functions b/bin/common-functions
index 222062e..d7f3e41 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -368,7 +368,7 @@ repository_of_package() {
 # official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending
 # print wether the specified package is an official package (print
 # $ending) or a community package (print 'community-$ending') or a
-# build-suppor package (print 'build-support')
+# build-support package (print 'build-support')
 
 official_or_community() {
   local prepo
-- 
cgit v1.2.3-54-g00ecf


From d2e0ef2a558e27e07150f239d91e4c29ce822426 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Feb 2018 11:14:03 +0100
Subject: bin/mysql-functions: mysql_join_*_* new

---
 bin/bootstrap-mysql                |  16 +---
 bin/build-master-status-from-mysql |  65 ++++++--------
 bin/check-bugtracker               |   2 +-
 bin/common-functions               |   5 +-
 bin/copy-to-build-support          |   6 +-
 bin/db-update                      |   4 +-
 bin/get-assignment                 |  34 ++++----
 bin/get-package-updates            |  43 ++++-----
 bin/modify-package-state           |  21 ++---
 bin/mysql-functions                | 174 +++++++++++++++++++++++++------------
 bin/prioritize-build-list          |   7 +-
 bin/return-assignment              |  40 ++++-----
 bin/seed-build-list                |  15 ++--
 bin/show-dependencies              |   7 +-
 bin/why-dont-you                   |  23 +++--
 15 files changed, 243 insertions(+), 219 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 322e0a1..30c260a 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -512,9 +512,8 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \
   ' | \
   while read -r state_file pkgbase git_revision mod_git_revision repository; do
     printf 'UPDATE `build_assignments`'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'package_sources'       'package_sources'       'build_assignments' 'package_source' \
-      'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
     printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \
       "$(
         tr -d '\n' < \
@@ -563,10 +562,9 @@ grep '^\('"$(
   "${tmp_dir}/new-stable-packages"
 
 if [ -s "${tmp_dir}/new-stable-packages" ]; then
+  # shellcheck disable=SC2016
   {
-    # shellcheck disable=SC2016
     printf 'CREATE TEMPORARY TABLE `stable_packages` ('
-    # shellcheck disable=SC2016
     printf '`%s` %s,' \
       'pkgname' 'VARCHAR(64)' \
       'epoch' 'MEDIUMINT' \
@@ -576,9 +574,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
       'architecture' 'VARCHAR(16)' \
       'repository' 'VARCHAR(64)' \
       'build_assignment' 'BIGINT NOT NULL AUTO_INCREMENT'
-    # shellcheck disable=SC2016
     printf 'PRIMARY KEY (`build_assignment`));\n'
-    # shellcheck disable=SC2016
     sed '
       1~10 ! b not_start
       s/^/INSERT IGNORE INTO `stable_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`) VALUES \n/
@@ -589,28 +585,22 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
       :end
       s/,$/;/
     ' "${tmp_dir}/new-stable-packages"
-    # shellcheck disable=SC2016
     printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)'
     printf ' SELECT '
-    # shellcheck disable=SC2016
     printf '`stable_packages`.`%s`,' \
       'pkgname' \
       'epoch' \
       'pkgver' \
       'pkgrel' \
       'sub_pkgrel'
-    # shellcheck disable=SC2016
     printf '`%s`.`id`,' \
       'architectures' \
       'repositories'
-    # shellcheck disable=SC2016
     printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`'
-    # shellcheck disable=SC2016
     printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \
       'repositories' 'repository' 'repositories' \
       'architectures' 'architecture' 'architectures'
     printf ';\n'
-    # shellcheck disable=SC2016
     printf 'DROP TABLE `stable_packages`;\n'
   } | \
     ${mysql_command}
diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 7a6605a..ea8d0f1 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -21,11 +21,10 @@ fi
 {
   printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`'
   printf ' FROM `package_sources`'
-  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-    'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \
-    'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
-    'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \
-    'repositories' 'binary_packages' 'repository' 'repositories'
+  mysql_join_package_sources_upstream_repositories
+  mysql_join_package_sources_build_assignments
+  mysql_join_build_assignments_binary_packages
+  mysql_join_binary_packages_repositories
   printf ' WHERE `repositories`.`name`="build-list"'
 } | \
   ${mysql_command} --batch | \
@@ -45,10 +44,9 @@ sort "${work_dir}/build-list" > \
     'pkgbase' 'git_revision' 'mod_git_revision'
   printf '`upstream_repositories`.`name`'
   printf ' FROM `build_slaves`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \
-    'package_sources' 'package_sources' 'build_assignments' 'package_source' \
-    'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+  mysql_join_build_slaves_build_assignments
+  mysql_join_build_assignments_package_sources
+  mysql_join_package_sources_upstream_repositories
 } | \
   ${mysql_command} --raw --batch | \
   sed '
@@ -68,10 +66,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin
 {
   printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
   printf ' FROM `package_sources`'
-  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-    'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
-    'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \
-    'repositories' 'binary_packages' 'repository' 'repositories'
+  mysql_join_package_sources_build_assignments
+  mysql_join_build_assignments_binary_packages
+  mysql_join_binary_packages_repositories
   printf ' WHERE `repositories`.`name`="deletion-list"'
   printf ' OR `binary_packages`.`is_to_be_deleted`;\n'
 } | \
@@ -89,9 +86,8 @@ sort "${work_dir}/deletion-list" > \
 {
   printf 'SELECT `package_sources`.`pkgbase`'
   printf ' FROM `package_sources`'
-  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-    'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
-    'build_dependency_loops' 'build_dependency_loops' 'build_assignment' 'build_assignments'
+  mysql_join_package_sources_build_assignments
+  mysql_join_build_assignments_build_dependency_loops
   printf ';\n'
 } | \
   ${mysql_command} --batch | \
@@ -118,10 +114,9 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
     'sub_pkgrel'
   printf '`architectures`.`name`'
   printf ' FROM `binary_packages`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'repositories'           'repositories'           'binary_packages' 'repository' \
-    'architectures'          'architectures'          'binary_packages' 'architecture' \
-    'repository_stabilities' 'repository_stabilities' 'repositories'    'stability'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  mysql_join_repositories_repository_stabilities
   printf ' WHERE `binary_packages`.`is_tested`'
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
@@ -154,10 +149,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     'sub_pkgrel'
   printf '`architectures`.`name`'
   printf ' FROM `binary_packages`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'repositories'           'repositories'           'binary_packages' 'repository' \
-    'architectures'          'architectures'          'binary_packages' 'architecture' \
-    'repository_stabilities' 'repository_stabilities' 'repositories'    'stability'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  mysql_join_repositories_repository_stabilities
   printf ' WHERE NOT `binary_packages`.`is_tested`'
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
@@ -188,15 +182,13 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     'mod_git_revision'
   printf '`upstream_repositories`.`name`,`install_targets`.`name`'
   printf ' FROM `binary_packages`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'dependencies'     'binary_packages'  'dependencies' 'dependent' \
-    'dependency_types' 'dependency_types' 'dependencies' 'dependency_type'
-  printf ' AND `dependency_types`.`relevant_for_building`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'install_targets'       'install_targets'       'dependencies'      'depending_on' \
-    'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
-    'package_sources'       'package_sources'       'build_assignments' 'package_source' \
-    'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+  mysql_join_binary_packages_dependencies
+  mysql_join_dependencies_dependency_types
+  mysql_join_dependencies_install_targets
+  mysql_join_binary_packages_build_assignments
+  mysql_join_build_assignments_package_sources
+  mysql_join_package_sources_upstream_repositories
+  printf ' WHERE `dependency_types`.`relevant_for_building`'
 } | \
   ${mysql_command} --raw --batch | \
   sed '
@@ -214,10 +206,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
       'pkgbase' 'git_revision' 'mod_git_revision'
     printf '`upstream_repositories`.`name`'
     printf ' FROM `upstream_repositories`'
-    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-      'package_sources'   'package_sources'   'upstream_package_repository' 'upstream_repositories' \
-      'build_assignments' 'build_assignments' 'package_source'              'package_sources' \
-      'binary_packages'   'binary_packages'   'build_assignment'            'build_assignments'
+    mysql_join_upstream_repositories_package_sources
+    mysql_join_package_sources_build_assignments
+    mysql_join_build_assignments_binary_packages
   } | \
     ${mysql_command} --raw --batch | \
     sed '
diff --git a/bin/check-bugtracker b/bin/check-bugtracker
index 72ab133..d6e92e5 100755
--- a/bin/check-bugtracker
+++ b/bin/check-bugtracker
@@ -31,7 +31,7 @@ bug_list=$(
   while read -r stability_id category; do
     for has_issues in '1:' '0:NOT '; do
       printf 'UPDATE `binary_packages`'
-      printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`'
+      mysql_join_binary_packages_repositories
       printf ' SET `has_issues`=%s' \
         "${has_issues%:*}"
       printf ' WHERE `repositories`.`stability`=%s' \
diff --git a/bin/common-functions b/bin/common-functions
index d7f3e41..a84271b 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -478,9 +478,8 @@ remove_old_package_versions() {
       printf '`binary_packages`.`sub_pkgrel`'
     printf ')'
     printf ' FROM `binary_packages`'
-    printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
-      'repositories'  'repositories'  'repository' \
-      'architectures' 'architectures' 'architecture'
+    mysql_join_binary_packages_repositories
+    mysql_join_binary_packages_architectures
     printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
     printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`'
     # name must match
diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 4710012..808d041 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -31,8 +31,8 @@ sed -n '
       {
         printf 'SELECT `binary_packages`.`id`,`repositories`.`name`'
         printf ' FROM `binary_packages`'
-        printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
-        printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+        mysql_join_binary_packages_repositories
+        mysql_join_binary_packages_architectures
         printf ' WHERE'
         printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
           'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
@@ -89,7 +89,7 @@ sed -n '
       # shellcheck disable=SC2086
       printf ',`binary_packages`.`%s`' ${to_copy}
       printf ' FROM `binary_packages`'
-      printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+      mysql_join_binary_packages_architectures
       printf ' WHERE'
       printf ' `binary_packages`.`id`=%s;\n' \
         "${id}"
diff --git a/bin/db-update b/bin/db-update
index e5ea193..b830981 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -325,8 +325,8 @@ move_packages() {
   while read -r package; do
     while read -r part; do
       printf 'UPDATE `binary_packages`'
-      printf ' JOIN `repositories` AS `from_repo` ON `binary_packages`.`repository`=`from_repo`.`id`'
-      printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+      mysql_join_binary_packages_repositories '' 'from_repo'
+      mysql_join_binary_packages_architectures
       printf ' SET `binary_packages`.`repository`=('
         printf 'SELECT `to_repo`.`id`'
         printf ' FROM `repositories` as `to_repo`'
diff --git a/bin/get-assignment b/bin/get-assignment
index a7eec2d..1edcffb 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -45,11 +45,10 @@ hand_out_assignment() {
       'pkgbase' 'git_revision' 'mod_git_revision'
     printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`'
     printf ' FROM `upstream_repositories`'
-    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-      'package_sources'   'package_sources'   'upstream_package_repository' 'upstream_repositories' \
-      'build_assignments' 'build_assignments' 'package_source'              'package_sources' \
-      'binary_packages'   'binary_packages'   'build_assignment'            'build_assignments' \
-      'repositories'      'binary_packages'   'repository'                  'repositories'
+    mysql_join_upstream_repositories_package_sources
+    mysql_join_package_sources_build_assignments
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
     printf ' WHERE `repositories`.`name`="build-list"'
     printf ' AND `package_sources`.`%s`=from_base64("%s")' \
       'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
@@ -80,8 +79,8 @@ hand_out_assignment() {
     printf ' SET `currently_building` = ('
     printf ' SELECT `build_assignments`.`id`'
     printf ' FROM `build_assignments`'
-    printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
-    printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
     printf ' WHERE'
     printf ' `package_sources`.`%s` = from_base64("%s") AND' \
       'pkgbase'          "$(printf '%s' "$1" | base64 -w0)" \
@@ -94,8 +93,8 @@ hand_out_assignment() {
       "$(printf '%s' "${slave}" | base64 -w0)"
 
     printf 'UPDATE `build_assignments`'
-    printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
-    printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
     printf ' SET `build_assignments`.`priority`=0'
     printf ' WHERE'
     printf ' `package_sources`.`%s` = from_base64("%s") AND' \
@@ -150,10 +149,9 @@ currently_building=$(
   {
     printf 'SELECT '
     mysql_query_select_pkgbase_and_revision
-    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-      'build_slaves'    'build_slaves'    'currently_building' 'build_assignments' \
-      'binary_packages' 'binary_packages' 'build_assignment'   'build_assignments' \
-      'repositories'    'binary_packages' 'repository'         'repositories'
+    mysql_join_build_assignments_build_slaves
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
     printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
       "$(printf '%s' "${slave}" | base64 -w0)"
     printf ' AND `repositories`.`name`="build-list"'
@@ -189,9 +187,8 @@ next_building=$(
     mysql_query_is_part_of_loop '`build_assignments`.`id`'
     printf ' AS `part_of_loop`,'
     mysql_query_select_pkgbase_and_revision
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \
-      'repositories' 'repositories' 'binary_packages' 'repository'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
     printf ' WHERE `repositories`.`name`="build-list"'
     printf ' AND NOT EXISTS ('
       printf ' SELECT *'
@@ -231,9 +228,8 @@ count_pending=$(
   {
     printf 'SELECT count(*)'
     printf ' FROM `build_assignments`'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \
-      'repositories' 'repositories' 'binary_packages' 'repository'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
     printf ' WHERE `repositories`.`name`="build-list"'
     printf ' AND `build_assignments`.`is_blocked` IS NULL'
     printf ';\n'
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 89b09df..e81d128 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -110,22 +110,18 @@ delete_package() {
     {
       # packages from the build-list/to-be-decided go straight to the deletion-list
       printf 'UPDATE `binary_packages`'
-      printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
-        'repositories' 'repositories' 'repository' \
-        'build_assignments' 'build_assignments' 'build_assignment'
-      printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-        'package_sources' 'package_sources' 'build_assignments' 'package_source'
+      mysql_join_binary_packages_repositories
+      mysql_join_binary_packages_build_assignments
+      mysql_join_build_assignments_package_sources
       printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
       printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")'
       printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \
         "$(printf '%s' "$1" | base64 -w0)"
       # other packages are marked as `is_to_be_deleted`
       printf 'UPDATE `binary_packages`'
-      printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
-        'repositories' 'repositories' 'repository' \
-        'build_assignments' 'build_assignments' 'build_assignment'
-      printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-        'package_sources' 'package_sources' 'build_assignments' 'package_source'
+      mysql_join_binary_packages_repositories
+      mysql_join_binary_packages_build_assignments
+      mysql_join_build_assignments_package_sources
       printf ' SET `binary_packages`.`is_to_be_deleted`=1'
       printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \
         "$(printf '%s' "$1" | base64 -w0)"
@@ -341,10 +337,9 @@ done | \
         # which are not yet built or on the deletion list
         printf 'DELETE `build_assignments`,`binary_packages`'
         printf ' FROM `binary_packages`'
-        printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-          'build_assignments' 'build_assignments' 'binary_packages'   'build_assignment' \
-          'package_sources'   'package_sources'   'build_assignments' 'package_source' \
-          'repositories'      'repositories'      'binary_packages'   'repository'
+        mysql_join_binary_packages_build_assignments
+        mysql_join_build_assignments_package_sources
+        mysql_join_binary_packages_repositories
         printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
           "$(
             printf '%s' "${package}" | \
@@ -353,10 +348,9 @@ done | \
         printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n'
         # remove is-to-be-deleted marker from old binary packages
         printf 'UPDATE `binary_packages`'
-        printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-          'build_assignments' 'build_assignments' 'binary_packages'   'build_assignment' \
-          'package_sources'   'package_sources'   'build_assignments' 'package_source' \
-          'repositories'      'repositories'      'binary_packages'   'repository'
+        mysql_join_binary_packages_build_assignments
+        mysql_join_build_assignments_package_sources
+        mysql_join_binary_packages_repositories
         printf ' SET `is_to_be_deleted`=0'
         printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \
           "$(
@@ -431,9 +425,9 @@ while [ -n "${black_listed_new}" ]; do
           '
         printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")'
         printf ' FROM `package_sources` AS `a_ps`'
-        printf ' JOIN `build_assignments` AS `a_ba` ON `a_ba`.`package_source`=`a_ps`.`id`'
-        printf ' JOIN `binary_packages` AS `a_bp` ON `a_bp`.`build_assignment`=`a_ba`.`id`'
-        printf ' JOIN `dependencies` ON `dependencies`.`dependent`=`a_bp`.`id`'
+        mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
+        mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
+        mysql_join_binary_packages_dependencies 'a_bp'
         printf ' WHERE NOT EXISTS ('
           printf ' SELECT *'
           printf ' FROM `install_target_providers`'
@@ -441,8 +435,8 @@ while [ -n "${black_listed_new}" ]; do
             printf ' SELECT *'
             printf ' FROM `bl`'
             printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
-            printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`'
-            printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`'
+            mysql_join_package_sources_build_assignments 'b_ps' 'b_ba'
+            mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp'
             printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
           printf ')'
           printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
@@ -607,8 +601,7 @@ fi
   done
   # move binary_packages from "to-be-decided" to "build-list"
   printf 'UPDATE `binary_packages`'
-  printf ' JOIN `repositories` AS `from_repo`'
-  printf ' ON `binary_packages`.`repository`=`from_repo`.`id`'
+  mysql_join_binary_packages_repositories '' 'from_repo'
   printf ' SET `repository`=('
     printf 'SELECT `to_repo`.`id`'
     printf ' FROM `repositories` AS `to_repo`'
diff --git a/bin/modify-package-state b/bin/modify-package-state
index b2d3133..b0ba53e 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -136,10 +136,9 @@ while read -r package reason; do
       # shellcheck disable=SC2016
       combiner=$(
         printf '`binary_packages`'
-        printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-          'repositories'           'repositories'           'binary_packages' 'repository' \
-          'repository_stabilities' 'repository_stabilities' 'repositories'    'stability' \
-          'architectures'          'architectures'          'binary_packages' 'architecture'
+        mysql_join_binary_packages_repositories
+        mysql_join_repositories_repository_stabilities
+        mysql_join_binary_packages_architectures
       )
       # shellcheck disable=SC2016,SC2031
       selector=$(
@@ -161,12 +160,11 @@ while read -r package reason; do
       # shellcheck disable=SC2016
       combiner=$(
         printf '`build_assignments`'
-        printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-          'package_sources'        'package_sources'        'build_assignments' 'package_source' \
-          'upstream_repositories'  'upstream_repositories'  'package_sources'   'upstream_package_repository' \
-          'binary_packages'        'build_assignments'      'binary_packages'   'build_assignment' \
-          'repositories'           'repositories'           'binary_packages'   'repository' \
-          'repository_stabilities' 'repository_stabilities' 'repositories'      'stability'
+        mysql_join_build_assignments_package_sources
+        mysql_join_package_sources_upstream_repositories
+        mysql_join_build_assignments_binary_packages
+        mysql_join_binary_packages_repositories
+        mysql_join_repositories_repository_stabilities
       )
       # shellcheck disable=SC2016
       selector=$(
@@ -238,8 +236,7 @@ while read -r package reason; do
       printf 'UPDATE `binary_packages` AS `a`'
       printf ' JOIN `binary_packages` AS `b`'
       printf ' ON `a`.`build_assignment`=`b`.`build_assignment`'
-      printf ' JOIN `repositories`'
-      printf ' ON `b`.`repository`=`repositories`.`id`'
+      mysql_join_binary_packages_repositories 'b'
       printf ' SET `b`.`is_tested`=1'
       printf ' WHERE `a`.`is_tested`'
       printf ' AND NOT `b`.`is_tested`'
diff --git a/bin/mysql-functions b/bin/mysql-functions
index 7f2cdab..6950ee8 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -131,7 +131,6 @@ mysql_show_binary_package() {
     shift
   done
 
-  local joint
   {
     printf 'SELECT'
     printf ' `%s`.`%s`,' \
@@ -146,17 +145,11 @@ mysql_show_binary_package() {
       'package_sources' 'mod_git_revision' \
       'upstream_repositories' 'name'
     printf ' FROM `binary_packages`'
-    for joint in \
-      'architectures:binary_packages:architecture' \
-      'package_sources:binary_packages:package_source' \
-      'repositories:binary_packages:repository' \
-      'upstream_repositories:package_sources:upstream_package_repository'; do
-      printf ' JOIN `%s` ON `%s`.`id` =' \
-        "${joint%%:*}" "${joint%%:*}"
-      joint="${joint#*:}"
-      printf ' `%s`.`%s`' \
-        "${joint%:*}" "${joint#*:}"
-    done
+    mysql_join_binary_packages_architectures
+    mysql_join_binary_packages_repositories
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
     printf ' WHERE'
     printf ' `%s`.`%s` = from_base64("%s") AND' \
       'binary_packages' 'pkgname' "${pkgname}" \
@@ -434,8 +427,8 @@ mysql_generate_package_metadata() {
             # build-list or in $current_repository (beware of split
             # packages!)
             printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
-            printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
-            printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+            mysql_join_binary_packages_architectures
+            mysql_join_binary_packages_repositories
             printf ' WHERE'
             printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
               'epoch' "${epoch}" \
@@ -448,7 +441,7 @@ mysql_generate_package_metadata() {
               "${current_repository}"
             # max(sub_pkgrel)+1
             printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
-            printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+            mysql_join_binary_packages_architectures
             printf ' WHERE'
             printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
               'epoch' "${epoch}" \
@@ -496,11 +489,9 @@ mysql_generate_package_metadata() {
           printf ' `%s` JOIN' \
             'repositories' \
             'architectures' \
-            'build_assignments' \
-            'package_sources'
-          printf ' ON `build_assignments`.`package_source` = `package_sources`.`id`'
-          printf ' JOIN `upstream_repositories`'
-          printf ' ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
+            'build_assignments'
+          mysql_join_build_assignments_package_sources
+          mysql_join_package_sources_upstream_repositories
           printf ' WHERE'
           printf ' `%s`.`%s` = from_base64("%s") AND' \
             'repositories' 'name' "${current_repository}" \
@@ -513,7 +504,6 @@ mysql_generate_package_metadata() {
         } | \
           sed '
             s|,)|)|g
-            s|JOIN ON|ON|g
             s| AND;$|;|
           ' >> \
           "${temp_dir}/add-binary-packages-command"
@@ -586,8 +576,7 @@ mysql_generate_package_metadata() {
               ;;
             esac
             printf ' `binary_packages`'
-            printf ' JOIN `architectures`'
-            printf ' ON `binary_packages`.`architecture` = `architectures`.`id`'
+            mysql_join_binary_packages_architectures
             printf ' JOIN `install_targets`'
             printf ' JOIN `%s`' "${link}"
             printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
@@ -649,9 +638,9 @@ mysql_sanity_check() {
   {
     printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)'
     printf ' FROM `binary_packages`'
-    printf ' JOIN `build_assignments` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
-    printf ' JOIN `architectures` as `bp_arch` ON `binary_packages`.`architecture`=`bp_arch`.`id`'
-    printf ' JOIN `architectures` as `ba_arch` ON `build_assignments`.`architecture`=`ba_arch`.`id`'
+    mysql_join_binary_packages_build_assignments
+    mysql_join_binary_packages_architectures '' 'bp_arch'
+    mysql_join_build_assignments_architectures '' 'ba_arch'
     printf ' WHERE `bp_arch`.`name`!="any"'
     printf ' AND `ba_arch`.`name`="any";\n'
     printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)'
@@ -660,14 +649,13 @@ mysql_sanity_check() {
     printf ' ON `a`.`pkgname`=`b`.`pkgname`'
     printf ' AND `a`.`repository`=`b`.`repository`'
     printf ' AND `a`.`id`!=`b`.`id`'
-    printf ' JOIN `repositories`'
-    printf ' ON `a`.`repository`=`repositories`.`id`'
+    mysql_join_binary_packages_repositories 'a'
     printf ' WHERE `repositories`.`name`="build-list";\n'
     printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)'
     printf ' FROM `binary_packages` AS `a`'
     printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`'
-    printf ' JOIN `repositories` AS `arep` ON `a`.`repository`=`arep`.`id`'
-    printf ' JOIN `repositories` AS `brep` ON `b`.`repository`=`brep`.`id`'
+    mysql_join_binary_packages_repositories 'a' 'arep'
+    mysql_join_binary_packages_repositories 'b' 'brep'
     printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`'
     printf ' AND `%srep`.`name`="build-list"' \
       'a' 'b'
@@ -700,8 +688,8 @@ mysql_sanity_check() {
     {
       printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`'
       printf ' FROM `binary_packages`'
-      printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
-      printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+      mysql_join_binary_packages_architectures
+      mysql_join_binary_packages_repositories
       printf ' WHERE `repositories`.`is_on_master_mirror`'
     } | \
       ${mysql_command} --batch | \
@@ -728,12 +716,11 @@ mysql_find_build_assignment_loops() {
     {
       printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`'
       printf ' FROM `dependencies`'
-      printf ' JOIN `install_target_providers` ON `dependencies`.`depending_on`=`install_target_providers`.`install_target`'
-      printf ' JOIN `binary_packages` `packages_dependent` ON `dependencies`.`dependent`=`packages_dependent`.`id`'
-      printf ' JOIN `binary_packages` `packages_dependency` ON `install_target_providers`.`package`=`packages_dependency`.`id`'
-      printf ' JOIN `repositories` `repositories_%s` ON `packages_%s`.`repository`=`repositories_%s`.`id`' \
-        'dependent' 'dependent' 'dependent' \
-        'dependency' 'dependency' 'dependency'
+      mysql_join_dependencies_install_target_providers
+      mysql_join_install_target_providers_binary_packages '' 'packages_dependency'
+      mysql_join_dependencies_binary_packages '' 'packages_dependent'
+      mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency'
+      mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent'
       printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
     } | \
       ${mysql_command} --raw --batch | \
@@ -836,8 +823,8 @@ mysql_cleanup() {
       printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`'
     printf ');\n'
     printf 'UPDATE `build_slaves`'
-    printf ' JOIN `binary_packages` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`'
-    printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`'
+    mysql_join_build_slaves_binary_packages
+    mysql_join_binary_packages_repositories
     printf ' SET `currently_building`=NULL'
     printf ' WHERE `repositories`.`name`!="build-list";\n'
   } | \
@@ -849,12 +836,11 @@ mysql_cleanup() {
 mysql_query_has_pending_dependencies() {
   printf 'EXISTS ('
     printf 'SELECT * FROM `binary_packages` as `to_dos`'
-    printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \
-      'repositories` AS `to_do_repos' 'to_do_repos'  'id'           'to_dos'                   'repository' \
-      'dependencies'                  'to_dos'       'id'           'dependencies'             'dependent' \
-      'install_target_providers'      'dependencies' 'depending_on' 'install_target_providers' 'install_target' \
-      'binary_packages` AS `bin_deps' 'bin_deps'     'id'           'install_target_providers' 'package' \
-      'repositories` AS `dep_repos'   'dep_repos'    'id'           'bin_deps'                 'repository'
+    mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos'
+    mysql_join_binary_packages_dependencies 'to_dos'
+    mysql_join_dependencies_install_target_providers
+    mysql_join_install_target_providers_binary_packages '' 'bin_deps'
+    mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos'
     printf ' WHERE'
     printf ' `%s`.`name`="build-list" AND' \
       'dep_repos' 'to_do_repos'
@@ -884,9 +870,8 @@ mysql_query_select_pkgbase_and_revision() {
     'mod_git_revision'
   printf '`upstream_repositories`.`name`'
   printf ' FROM `build_assignments`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'package_sources' 'package_sources' 'build_assignments' 'package_source' \
-    'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
+  mysql_join_build_assignments_package_sources
+  mysql_join_package_sources_upstream_repositories
 }
 
 # mysql_repair_binary_packages_without_build_assignment
@@ -900,7 +885,7 @@ mysql_repair_binary_packages_without_build_assignment() {
       '`binary_packages`.`pkgname`' \
       '`architectures`.`name`'
     printf ' FROM `binary_packages`'
-    printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+    mysql_join_binary_packages_architectures
     printf ' WHERE `binary_packages`.`build_assignment`<0'
   } | \
     ${mysql_command} --raw --batch | \
@@ -954,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() {
       printf ' LIMIT 1;\n'
       printf 'UPDATE `binary_packages`'
       printf ' JOIN `build_assignments`'
-      printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+      mysql_join_binary_packages_build_assignments
       printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`'
       printf ' WHERE `binary_packages`.`id`=%s' "${id}"
       printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}"
@@ -980,9 +965,8 @@ mysql_remove_duplicate_build_order() {
       'pkgver' 'pkgver' \
       'pkgrel' 'pkgrel'
     printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`'
-    printf ' JOIN `repositories` AS `%s` ON `%s`.`id`=`%s`.`repository`' \
-      'orep' 'orep' 'old' \
-      'nrep' 'nrep' 'new'
+    mysql_join_binary_packages_repositories 'old' 'orep'
+    mysql_join_binary_packages_repositories 'new' 'nrep'
     printf ' WHERE `orep`.`name`!="to-be-decided"'
     printf ' AND `nrep`.`name`="to-be-decided";\n'
     printf 'UPDATE IGNORE `dependencies`'
@@ -1004,6 +988,9 @@ mysql_remove_duplicate_build_order() {
     ${mysql_command}
 }
 
+# mysql_package_name_query
+# print a mysql query of the full name of a package file
+
 mysql_package_name_query() {
   printf 'CONCAT('
   printf '`binary_packages`.`pkgname`,"-",'
@@ -1014,3 +1001,82 @@ mysql_package_name_query() {
   printf '`architectures`.`name`,".pkg.tar.xz"'
   printf ')'
 }
+
+# mysql_join_*_*
+# print 'JOIN' part of mysql query to connect the respective tables
+# these functions take 2 optional arguments, acting as aliases for
+# the tables
+
+# mysql_join__generic $table_a $column_a $table_b $column_b
+# create mysql_join_${table_a}_${table_b}() function
+
+mysql_join__generic() {
+  eval "$(
+    printf 'mysql_join_%s_%s() {\n' "$1" "$3"
+    printf '  printf '"'"' JOIN `%s`'"'"'\n' "$3"
+    printf '  if [ -n "$2" ]; then\n'
+    printf '    printf '"'"' AS `%%s`'"'"' "$2"\n'
+    printf '  fi\n'
+    printf '  if [ -n "$1" ]; then\n'
+    printf '    printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2"
+    printf '  else\n'
+    printf '    printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2"
+    printf '  fi\n'
+    printf '  if [ -n "$2" ]; then\n'
+    printf '    printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4"
+    printf '  else\n'
+    printf '    printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4"
+    printf '  fi\n'
+    printf '}\n'
+  )"
+}
+
+for link in \
+  'binary_packages:architecture:architectures' \
+  'binary_packages:repository:repositories' \
+  'binary_packages:build_assignment:build_assignments' \
+ \
+  'build_assignments:architecture:architectures' \
+  'build_assignments:package_source:package_sources' \
+ \
+  'build_dependency_loops:build_assignment:build_assignments' \
+  'build_dependency_loops:build_assignment build_assignment:binary_packages' \
+ \
+  'build_slaves:currently_building:build_assignments' \
+  'build_slaves:currently_building build_assignment:binary_packages' \
+ \
+  'dependencies:depending_on:install_targets' \
+  'dependencies:dependent:binary_packages' \
+  'dependencies:dependency_type:dependency_types' \
+ \
+  'failed_builds:reason:fail_reason' \
+  'failed_builds:build_assignment:build_assignments' \
+  'failed_builds:build_slave:build_slaves' \
+ \
+  'install_target_providers:package:binary_packages' \
+  'install_target_providers:install_target:install_targets' \
+  'install_target_providers:install_target depending_on:dependencies' \
+ \
+  'package_sources:upstream_package_repository:upstream_repositories' \
+ \
+  'repositories:stability:repository_stabilities' \
+ \
+  'upstream_repositories:git_repository:git_repositories'; do
+# A join for these cannot be done, because it's not clear on what to join:
+#  'repository_stability_relations:more_stable:repository_stabilities'
+#  'repository_stability_relations:less_stable:repository_stabilities'
+
+  table_b="${link##*:}"
+  table_a="${link%:*}"
+  column_b="${table_a##*:}"
+  table_a="${table_a%:*}"
+  column_a="${column_b% *}"
+  if [ "${column_a}" = "${column_b}" ]; then
+    column_b='id'
+  else
+    column_b="${column_b##* }"
+  fi
+
+  mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}"
+  mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}"
+done
diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list
index d611e79..e8dcf0a 100755
--- a/bin/prioritize-build-list
+++ b/bin/prioritize-build-list
@@ -32,10 +32,9 @@ fi
 # shellcheck disable=SC2016
 {
   printf 'UPDATE `build_assignments`'
-  printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-    'package_sources' 'build_assignments' 'package_source'   'package_sources' \
-    'binary_packages' 'binary_packages'   'build_assignment' 'build_assignments' \
-    'repositories'    'binary_packages'   'repository'       'repositories'
+  mysql_join_build_assignments_package_sources
+  mysql_join_build_assignments_binary_packages
+  mysql_join_binary_packages_repositories
   printf ' SET `build_assignments`.`priority`=('
     printf 'SELECT COALESCE(MAX(`all_priorities`.`priority`),0)+1'
     printf ' FROM ('
diff --git a/bin/return-assignment b/bin/return-assignment
index 7b2dec3..995eae4 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -51,12 +51,11 @@ if [ "$5" = 'ERROR' ]; then
   infos=$(
     {
       printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`'
-      printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-        'build_assignments'     'build_assignments'     'build_slaves'      'currently_building' \
-        'package_sources'       'package_sources'       'build_assignments' 'package_source' \
-        'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository' \
-        'binary_packages'       'build_assignments'     'binary_packages'   'build_assignment' \
-        'repositories'          'repositories'          'binary_packages'   'repository'
+      mysql_join_build_assignments_build_slaves
+      mysql_join_build_assignments_package_sources
+      mysql_join_package_sources_upstream_repositories
+      mysql_join_build_assignments_binary_packages
+      mysql_join_binary_packages_repositories
       printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
         "$(
           # shellcheck disable=SC2154
@@ -157,7 +156,7 @@ if [ "$5" = 'ERROR' ]; then
       printf 'DROP TABLE `failures`;\n'
     fi
     printf 'UPDATE `build_assignments`'
-    printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
+    mysql_join_build_assignments_build_slaves
     printf ' SET `build_assignments`.`is_broken`=1, `build_slaves`.`currently_building`=NULL'
     printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
       "$(
@@ -267,12 +266,11 @@ clean_up_lock_file() {
 if ! {
     printf 'SELECT count(*)'
     printf ' FROM `binary_packages`'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
-      'build_slaves'          'build_assignments'     'build_slaves'      'currently_building' \
-      'repositories'          'repositories'          'binary_packages'   'repository' \
-      'package_sources'       'package_sources'       'build_assignments' 'package_source' \
-      'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_build_slaves
+    mysql_join_binary_packages_repositories
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
     printf ' WHERE `repositories`.`name`="build-list"'
     printf ' AND `build_slaves`.`name`=from_base64("%s")' \
       "$(printf '%s' "${slave}" | base64 -w0)"
@@ -396,9 +394,9 @@ package_errors=$(
         mysql_package_name_query
       printf ')'
       printf ' FROM `binary_packages`'
-      printf ' JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`'
-      printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`'
-      printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+      mysql_join_binary_packages_architectures
+      mysql_join_binary_packages_build_slaves
+      mysql_join_binary_packages_repositories
       printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
         "$(
           printf '%s' "${slave}" | \
@@ -476,9 +474,9 @@ fi
 # shellcheck disable=SC2016
 {
   printf 'UPDATE `build_assignments`'
-  printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
-  printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
-  printf ' JOIN `repositories` AS `old_repo` ON `binary_packages`.`repository`=`old_repo`.`id`'
+  mysql_join_build_assignments_build_slaves
+  mysql_join_build_assignments_binary_packages
+  mysql_join_binary_packages_repositories '' 'old_repo'
   printf ' SET'
   printf ' `build_assignments`.`is_broken`=0,'
   printf ' `build_assignments`.`priority`=0,'
@@ -507,8 +505,8 @@ fi
   printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n'
   printf 'INSERT INTO `loops_to_delete`'
   printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`'
-  printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_dependency_loops`.`build_assignment`'
-  printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+  mysql_join_build_dependency_loops_binary_packages
+  mysql_join_binary_packages_repositories
   printf ' WHERE NOT `repositories`.`name` = "build-list";\n'
   printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS ('
   printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`'
diff --git a/bin/seed-build-list b/bin/seed-build-list
index bfe975e..5ce9037 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -123,7 +123,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then
       printf 'SELECT '
       mysql_package_name_query
       printf ' FROM `binary_packages`'
-      printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
+      mysql_join_binary_packages_architectures
     } | \
       ${mysql_command} --raw --batch | \
       sed '
@@ -161,7 +161,7 @@ fi
   printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)'
   printf ' SELECT DISTINCT `binary_packages`.`pkgname`'
   printf ' FROM `binary_packages`'
-  printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
+  mysql_join_binary_packages_repositories
   printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list");\n'
 } | \
   sponge "${tmp_dir}/ignore-packages"
@@ -253,7 +253,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
   printf '`pkgbases`.`repository`'
   printf ' FROM `pkgbases`'
   printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`'
-  printf ' JOIN `git_repositories` ON `git_repositories`.`id`=`upstream_repositories`.`git_repository`'
+  mysql_join_upstream_repositories_git_repositories
   printf ';\n'
   printf 'SELECT '
   printf '`package_sources`.`pkgbase`,'
@@ -264,11 +264,10 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
   printf '),'
   printf '`upstream_repositories`.`name`'
   printf ' FROM `binary_packages`'
-  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-    'build_assignments'     'build_assignments'     'binary_packages'       'build_assignment' \
-    'package_sources'       'package_sources'       'build_assignments'     'package_source' \
-    'upstream_repositories' 'upstream_repositories' 'package_sources'       'upstream_package_repository' \
-    'git_repositories'      'git_repositories'      'upstream_repositories' 'git_repository'
+  mysql_join_binary_packages_build_assignments
+  mysql_join_build_assignments_package_sources
+  mysql_join_package_sources_upstream_repositories
+  mysql_join_upstream_repositories_git_repositories
   printf ' WHERE ('
   if [ -s "${tmp_dir}/package-regexes" ]; then
     grep -vxF '' "${tmp_dir}/package-regexes" | \
diff --git a/bin/show-dependencies b/bin/show-dependencies
index 0a7a1fd..bb1ae37 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -13,7 +13,7 @@ if [ $# -eq 0 ]; then
     {
       printf 'SELECT `package_sources`.`pkgbase`'
       printf ' FROM `package_sources`'
-      printf ' JOIN `build_assignments` ON `build_assignments`.`package_source`=`package_sources`.`id`'
+      mysql_join_package_sources_build_assignments
       printf ' WHERE `build_assignments`.`is_broken`'
     } | \
       ${mysql_command} --raw --batch | \
@@ -28,9 +28,8 @@ if [ $# -eq 0 ]; then
   {
     printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`binary_packages`.`pkgname`'
     printf ' FROM `binary_packages`'
-    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
-      'build_assignments' 'build_assignments' 'binary_packages'   'build_assignment' \
-      'package_sources'   'package_sources'   'build_assignments' 'package_source'
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
     printf ' ORDER BY `binary_packages`.`pkgname`'
     # Why can't mysql order in the same way "sort" does!
   } | \
diff --git a/bin/why-dont-you b/bin/why-dont-you
index e9922b7..d95f2a7 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -57,10 +57,9 @@ case "${action}" in
         # package_source, build_assignment, binary_package, repostory
         printf 'SELECT DISTINCT `tb_ps`.`pkgbase`,`tb_bin`.`id` AS `bin_id`,`tb_ba`.`id` AS `ba_id`,`tb_ba`.`is_blocked`,`tb_ba`.`is_broken`'
         printf ' FROM `package_sources` AS `tb_ps`'
-        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-          'build_assignments` AS `tb_ba' 'tb_ba'  'package_source'   'tb_ps' \
-          'binary_packages` AS `tb_bin'  'tb_bin' 'build_assignment' 'tb_ba' \
-          'repositories` AS `tb_rep'     'tb_bin' 'repository'       'tb_rep'
+        mysql_join_package_sources_build_assignments 'tb_ps' 'tb_ba'
+        mysql_join_build_assignments_binary_packages 'tb_ba' 'tb_bin'
+        mysql_join_binary_packages_repositories 'tb_bin' 'tb_rep'
         printf ' WHERE `tb_rep`.`name`="build-list"'
         printf ') AS `to_build`'
       printf ' ON `to_build`.`pkgbase`=`pkgbases`.`pkgbase`'
@@ -70,16 +69,14 @@ case "${action}" in
         # potential dependencies
         printf 'SELECT DISTINCT `dep_ps`.`pkgbase`,`dependencies`.`dependent`'
         printf ' FROM `package_sources` AS `dep_ps`'
-        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-          'build_assignments` AS `dep_ba' 'dep_ba'                   'package_source'   'dep_ps' \
-          'binary_packages` AS `dep_bin'  'dep_bin'                  'build_assignment' 'dep_ba' \
-          'repositories` AS `dep_rep'     'dep_bin'                  'repository'       'dep_rep' \
-          'install_target_providers'      'install_target_providers' 'package'          'dep_bin'
-        # starting from the line above, we have some additional joins,
+        mysql_join_package_sources_build_assignments 'dep_ps' 'dep_ba'
+        mysql_join_build_assignments_binary_packages 'dep_ba' 'dep_bin'
+        mysql_join_binary_packages_repositories 'dep_bin' 'dep_rep'
+        # now we have some (=3) additional joins,
         # because we are interested in dependency relations to `to_build`
-        printf ' JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
-        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
-          'dependency_types' 'dependencies' 'dependency_type' 'dependency_types'
+        mysql_join_binary_packages_install_target_providers 'dep_bin'
+        mysql_join_install_target_providers_dependencies
+        mysql_join_dependencies_dependency_types
         printf ' WHERE `dep_rep`.`name`="build-list"'
         printf ' AND `dependency_types`.`relevant_for_building`'
         printf ') AS `deps`'
-- 
cgit v1.2.3-54-g00ecf


From a9bb4512da90d21fde9a6f6095e304f461fe2dc7 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Feb 2018 11:33:29 +0100
Subject: bin/bootstrap-mysql: build-support should superseed build-support,
 too

---
 bin/bootstrap-mysql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 30c260a..4dee0f8 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -418,7 +418,8 @@ fi
       'staging' 'staging' \
       'staging' 'unbuilt' \
       'staging' 'forbidden' \
-      'unbuilt' 'forbidden'
+      'unbuilt' 'forbidden' \
+      'build-support' 'build-support'
     printf ';\n'
   } | \
     sed 's| OR ;|;|'
-- 
cgit v1.2.3-54-g00ecf


From 768316ba9c808e751094869080a2dd67f7848bda Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Feb 2018 11:39:08 +0100
Subject: bin/mysql-functions: fix typo in mysql query

---
 bin/mysql-functions | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 6950ee8..9e00887 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -504,6 +504,7 @@ mysql_generate_package_metadata() {
         } | \
           sed '
             s|,)|)|g
+            s| JOIN JOIN | JOIN |
             s| AND;$|;|
           ' >> \
           "${temp_dir}/add-binary-packages-command"
-- 
cgit v1.2.3-54-g00ecf


From d8c484b2ca5ff4cc7f5eb642b5ca134733460e70 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Feb 2018 11:58:03 +0100
Subject: bin/db-update: start reading infos from the database (for now, only
 the sanity-check)

---
 bin/db-update | 98 +++++++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 76 insertions(+), 22 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index b830981..de263be 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -439,29 +439,83 @@ fi
 tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-for package in ${packages_to_force_stabilize}; do
-  # some sanity checks
-  if [ ! -f "${work_dir}/package-states/${package}.tested" ] && \
-    [ ! -f "${work_dir}/package-states/${package}.testing" ] && \
-    [ ! -f "${work_dir}/package-states/${package}.done" ] && \
-    ! tr ' ' '.' < \
-      "${work_dir}/build-list" | \
-      grep -qxF "${package}"; then
-    >&2 printf 'Package "%s" is not in testing, staging or on the build list!\n' "${package}"
-    exit 2
-  fi
-done
+errors=$(
+  {
+    # shellcheck disable=SC2016
+    {
+      printf 'SELECT DISTINCT CONCAT('
+      printf '`package_sources`.`%s`,".",' \
+        'pkgbase' 'git_revision' 'mod_git_revision'
+      printf '`upstream_repositories`.`name`)'
+      printf ' FROM `binary_packages`'
+      mysql_join_binary_packages_repositories
+      mysql_join_binary_packages_build_assignments
+      mysql_join_build_assignments_package_sources
+      mysql_join_package_sources_upstream_repositories
+      # consider all packages less stable than "testing"
+      printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
+      printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
+      printf ' WHERE `repository_stabilities`.`name` = "testing"'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '
+        1d
+        p
+      '
+    # shellcheck disable=SC2086
+    printf '%s\n' ${packages_to_force_stabilize} | \
+      sort -u
+  } | \
+    grep -vxF '' | \
+    sort | \
+    uniq -u
+)
+if [ -n "${errors}" ]; then
+  # shellcheck disable=SC2086
+  >&2 printf 'Package "%s" is not in testing, staging or on the build list!\n' ${errors}
+  # shellcheck disable=SC2016
+  >&2 printf 'Did you really give it in the form "$pkgbase.$git_rev.$mod_git_rev.$repository"?\n'
+  exit 2
+fi
 
-for package in ${packages_to_force_unstage}; do
-  # some sanity checks
-  if [ ! -f "${work_dir}/package-states/${package}.done" ] && \
-    ! tr ' ' '.' < \
-      "${work_dir}/build-list" | \
-      grep -qxF "${package}"; then
-    >&2 printf 'Package "%s" is not in staging or on the build list!\n' "${package}"
-    exit 2
-  fi
-done
+errors=$(
+  {
+    # shellcheck disable=SC2016
+    {
+      printf 'SELECT DISTINCT CONCAT('
+      printf '`package_sources`.`%s`,".",' \
+        'pkgbase' 'git_revision' 'mod_git_revision'
+      printf '`upstream_repositories`.`name`)'
+      printf ' FROM `binary_packages`'
+      mysql_join_binary_packages_repositories
+      mysql_join_binary_packages_build_assignments
+      mysql_join_build_assignments_package_sources
+      mysql_join_package_sources_upstream_repositories
+      # consider all packages less stable than "staging"
+      printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
+      printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
+      printf ' WHERE `repository_stabilities`.`name` = "staging"'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '
+        1d
+        p
+      '
+    # shellcheck disable=SC2086
+    printf '%s\n' ${packages_to_force_unstage} | \
+      sort -u
+  } | \
+    grep -vxF '' | \
+    sort | \
+    uniq -u
+)
+if [ -n "${errors}" ]; then
+  # shellcheck disable=SC2086
+  >&2 printf 'Package "%s" is not in staging or on the build list!\n' ${errors}
+  # shellcheck disable=SC2016
+  >&2 printf 'Did you really give it in the form "$pkgbase.$git_rev.$mod_git_rev.$repository"?\n'
+  exit 2
+fi
 
 # Create a lock file and a trap.
 
-- 
cgit v1.2.3-54-g00ecf


From a4cc68010b9174feef5854fe667079be3d5785dc Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Feb 2018 15:27:19 +0100
Subject: bin/build-packages: make list of provided libraries

---
 bin/build-packages | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/bin/build-packages b/bin/build-packages
index 7f009c3..b399163 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -392,12 +392,25 @@ while [ "${count}" -ne 0 ]; do
                   >&2 printf ' failed. Next ...\n'
                 done
             done
+          >&2 printf 'searching for provided libraries\n'
+          find "${tar_content_dir}" -maxdepth 1 \
+            -name '*.pkg.tar.xz' \
+            -printf '%p\n' | \
+            while read -r pkgfile; do
+              pacman -Qqlp "${pkgfile}" | \
+                sed -n '
+                  s,^.*/,,
+                  /\.so\(\..\+\)\?$/p
+                ' > \
+                "${pkgfile}.so.provides"
+            done
+          # TODO: search for required libraries, too
+          >&2 printf 'running namcap ...'
           if [ "${repository}" = 'multilib' ]; then
             x86_64_build_command='multilib-build'
           else
             x86_64_build_command='extra-x86_64-build'
           fi
-          >&2 printf 'running namcap ...'
           # this is a little hack: makepkg receives '--version', but namcap is run nevertheless
           # (and it only works with devtools32, because they are running namcap on *.pkg.tar.xz in the base directory, too)
           sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \
@@ -429,9 +442,16 @@ while [ "${count}" -ne 0 ]; do
             done
           >&2 printf ' ok.\n'
           if ${upload_to_build_master}; then
-            find "${tar_content_dir}/" -maxdepth 1 -name '*.pkg.tar.xz-namcap.log' -execdir gzip '{}' \;
+            find "${tar_content_dir}/" -maxdepth 1 \
+              \( \
+                -name '*.pkg.tar.xz-namcap.log' -o \
+                -name '*.pkg.tar.xz.so.provides' \
+              \) \
+              -execdir gzip '{}' \;
           else
-            find "${tar_content_dir}/" -maxdepth 1 -name '*.pkg.tar.xz-namcap.log' -execdir grep -HF '' '{}' \;
+            find "${tar_content_dir}/" -maxdepth 1 \
+              -name '*.pkg.tar.xz-namcap.log' \
+              -execdir grep -HF '' '{}' \;
           fi
           # shellcheck disable=SC2046
           tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
-- 
cgit v1.2.3-54-g00ecf


From 829a12da2951daf1454f5f02eb8b02f3ac675e91 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Feb 2018 16:16:53 +0100
Subject: bin/get-assignment: add TODO

---
 bin/get-assignment | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 1edcffb..083b162 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -166,9 +166,14 @@ if [ -n "${currently_building}" ]; then
   hand_out_assignment ${currently_building}
 fi
 
-# a package with all dependencies met or which is part of a loop,
-#   that we requested or
-#   which has been built the least times
+# TODO: actually implement the below criteria (a.k.a.: "I'm out of time")
+
+# a package with [all dependencies met or which is part of a loop]
+# and which is currently not being built, ordered by:
+# 1: we requested it
+# 2: its priority
+# 3: is not yet built
+# 4: was built the longest time ago
 next_building=$(
   # shellcheck disable=SC2016
   {
-- 
cgit v1.2.3-54-g00ecf


From 568b87eba56a6fb4f05be0410a8f04a2341c8628 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 17 Feb 2018 19:35:21 +0100
Subject: bin/return-assignment: join in correct direction

---
 bin/return-assignment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 995eae4..6f96ee0 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -51,7 +51,7 @@ if [ "$5" = 'ERROR' ]; then
   infos=$(
     {
       printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`'
-      mysql_join_build_assignments_build_slaves
+      mysql_join_build_slaves_build_assignments
       mysql_join_build_assignments_package_sources
       mysql_join_package_sources_upstream_repositories
       mysql_join_build_assignments_binary_packages
-- 
cgit v1.2.3-54-g00ecf


From 33f10c6217830fd2430429da1d7f2d7a451522be Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 17 Feb 2018 20:27:43 +0100
Subject: bin/get-package-updates: save correct revisions in database

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index e81d128..d516a96 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -595,7 +595,7 @@ fi
   for repo in ${repo_names}; do
     printf 'UPDATE `git_repositories`'
     printf ' SET `git_repositories`.`head`=from_base64("%s")' \
-      "$(eval 'printf '"'"'%s'"'"' "${repo_revisions__'"${repo}"'}" | base64 -w0')"
+      "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}" | base64 -w0')"
     printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
       "$(printf '%s' "${repo}" | base64 -w0)"
   done
-- 
cgit v1.2.3-54-g00ecf


From ba935ea9e7709c284bfe714db156f17eea75819a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 17 Feb 2018 21:21:32 +0100
Subject: bin/build-packages: put *.so.provides.gz into tar, too

---
 bin/build-packages | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index b399163..97d74d1 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -455,7 +455,14 @@ while [ "${count}" -ne 0 ]; do
           fi
           # shellcheck disable=SC2046
           tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
-            find "${tar_content_dir}/" -maxdepth 1 \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' -o -name '*.pkg.tar.xz-namcap.log.gz' \) -printf '%f\n'
+            find "${tar_content_dir}/" -maxdepth 1 \
+              \( \
+                -name '*.pkg.tar.xz' -o \
+                -name '*.pkg.tar.xz.sig' -o \
+                -name '*.pkg.tar.xz-namcap.log.gz' \
+                -name '*.pkg.tar.xz.so.provides.gz' \
+              \) \
+              -printf '%f\n'
             )
           while ${upload_to_build_master}; do
             err=0
-- 
cgit v1.2.3-54-g00ecf


From 8fe54b099f881a3dbe885f3c29049324b2ae4747 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 08:25:51 +0100
Subject: bin/mysql-functions: small cleanup

---
 bin/mysql-functions | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 9e00887..5850ab5 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -725,6 +725,10 @@ mysql_find_build_assignment_loops() {
       printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
     } | \
       ${mysql_command} --raw --batch | \
+      sed '
+        1d
+        y/\t/ /
+      ' | \
       tsort 2>&1 >/dev/null | \
       sed 's/^tsort:\s*//' | \
       {
-- 
cgit v1.2.3-54-g00ecf


From 89745907f5137c4de7ed7dfc50646b066cd8fa86 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 08:27:48 +0100
Subject: bin/get-package-updates: clarify comment

---
 bin/get-package-updates | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index d516a96..97422f5 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -611,7 +611,8 @@ fi
 } | \
   ${mysql_command}
 
-# update loop list in database
+# update loop list in database (beware, the packages are expected to be in "build-list",
+# not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list".
 mysql_find_build_assignment_loops
 
 # Move the .new-files to the actual files
-- 
cgit v1.2.3-54-g00ecf


From b8b37df3177987401574462fe17c3be9a9ed94f7 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 14:25:04 +0100
Subject: bin/build-packages: fix typo

---
 bin/build-packages | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index 97d74d1..b6c4a7a 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -459,7 +459,7 @@ while [ "${count}" -ne 0 ]; do
               \( \
                 -name '*.pkg.tar.xz' -o \
                 -name '*.pkg.tar.xz.sig' -o \
-                -name '*.pkg.tar.xz-namcap.log.gz' \
+                -name '*.pkg.tar.xz-namcap.log.gz' -o \
                 -name '*.pkg.tar.xz.so.provides.gz' \
               \) \
               -printf '%f\n'
-- 
cgit v1.2.3-54-g00ecf


From 7bc95ff14d099110736feb6298aedf07bc6b67a6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 20:22:29 +0100
Subject: bin/get-assignment: hand out jobs by last-trial date, not by
 trial-count

---
 bin/get-assignment | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 083b162..3dbd386 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -166,8 +166,6 @@ if [ -n "${currently_building}" ]; then
   hand_out_assignment ${currently_building}
 fi
 
-# TODO: actually implement the below criteria (a.k.a.: "I'm out of time")
-
 # a package with [all dependencies met or which is part of a loop]
 # and which is currently not being built, ordered by:
 # 1: we requested it
@@ -184,16 +182,16 @@ next_building=$(
           base64 -w0
       )"
     printf '`build_assignments`.`priority`,'
-    printf '('
-      printf 'SELECT COUNT(*)'
-      printf ' FROM `failed_builds`'
-      printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
-    printf ') AS `trials`,'
+    printf 'COALESCE('
+    printf 'MAX(`failed_builds`.`date`),0'
+    printf ') AS `last_trial`,'
     mysql_query_is_part_of_loop '`build_assignments`.`id`'
     printf ' AS `part_of_loop`,'
     mysql_query_select_pkgbase_and_revision
     mysql_join_build_assignments_binary_packages
     mysql_join_binary_packages_repositories
+    printf ' LEFT'
+    mysql_join_build_assignments_failed_builds
     printf ' WHERE `repositories`.`name`="build-list"'
     printf ' AND NOT EXISTS ('
       printf ' SELECT *'
@@ -212,7 +210,8 @@ next_building=$(
       printf ' OR NOT '
       mysql_query_has_pending_dependencies '`build_assignments`.`id`'
     printf ')'
-    printf ' ORDER BY `requested` DESC, `priority` DESC, `trials`, `part_of_loop`, `build_assignments`.`id`'
+    printf ' GROUP BY `build_assignments`.`id`'
+    printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
   } | \
     ${mysql_command} --batch --raw | \
-- 
cgit v1.2.3-54-g00ecf


From fd812f6973156991a4462ef996161730dfad9e78 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 20:34:43 +0100
Subject: bin/get-package-updates: remove useless cat - we have this in a
 variable already

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 97422f5..1ce72e5 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -379,7 +379,7 @@ black_listed=''
 black_listed_new=$(
   {
     {
-      git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \
+      git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
         tar -Ox 'blacklist' | \
         sed '
           s/\s*#.*$//
-- 
cgit v1.2.3-54-g00ecf


From ee2db3dfee0b1aebbadbc9d54281b69e226c4cef Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 20:39:15 +0100
Subject: bin/seed-build-list: we should call generate_package_metadata on all
 packages which are seeded into the build-list via
 mysql_generate_package_metadata

---
 bin/seed-build-list | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 5ce9037..aff7acb 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -302,6 +302,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
 
     while read -r pkgbase git_rev mod_git_rev repo; do
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
+      generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       printf '\n' >&2
     done
-- 
cgit v1.2.3-54-g00ecf


From 1dca50324cdeb4c88460cd59b5684705c8e5b70a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 20:58:12 +0100
Subject: bin/mysql-functions: remove binary_packages from to-be-decided, style

---
 bin/mysql-functions | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/bin/mysql-functions b/bin/mysql-functions
index 5850ab5..185d9a9 100755
--- a/bin/mysql-functions
+++ b/bin/mysql-functions
@@ -771,6 +771,15 @@ mysql_cleanup() {
     exit 2
   fi
   {
+    # remove to-be-decided binary_packages
+    printf '%s ' \
+      "${operator}"
+    if [ "${operator}" = 'DELETE' ]; then
+      printf '`binary_packages` '
+    fi
+    printf 'FROM `binary_packages`'
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="to-be-decided";\n'
     # remove dependencies w/o binary_package or install_target
     printf '%s FROM `dependencies` ' \
       "${operator}"
@@ -804,13 +813,15 @@ mysql_cleanup() {
       printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`'
     printf ');\n'
     # remove failed_builds with unbroken build_assignments
-    printf '%s FROM `failed_builds` ' \
+    printf '%s ' \
       "${operator}"
-    printf 'WHERE ('
-      printf 'SELECT NOT `build_assignments`.`is_broken`'
-      printf ' FROM `build_assignments` '
-      printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
-    printf ');\n'
+    if [ "${operator}" = 'DELETE' ]; then
+      printf '`failed_builds` '
+    fi
+    printf 'FROM `failed_builds` '
+    mysql_join_failed_builds_build_assignments
+    printf 'WHERE NOT `build_assignments`.`is_broken`'
+    printf ';\n'
     # remove failed_builds w/o build_assignment
     printf '%s FROM `failed_builds` ' \
       "${operator}"
-- 
cgit v1.2.3-54-g00ecf


From f05923eda5d3a307f7fbfb44e451fd9441222aae Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 21:03:16 +0100
Subject: bin/get-package-updates: use mysql_cleanup

---
 bin/get-package-updates | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1ce72e5..70f62e8 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -196,16 +196,8 @@ if ! flock -s ${block_flag} 8; then
   exit
 fi
 
-# shellcheck disable=SC2016
-{
-  printf 'DELETE FROM `binary_packages`'
-  printf ' WHERE `binary_packages`.`repository`=('
-    printf 'SELECT `repositories`.`id`'
-    printf ' FROM `repositories`'
-    printf ' WHERE `repositories`.`name`="to-be-decided"'
-  printf ')'
-} | \
-  ${mysql_command}
+# shellcheck disable=SC2119
+mysql_cleanup
 
 echo 'Check modified packages from the last update, and put them to the build list.'
 
-- 
cgit v1.2.3-54-g00ecf


From fab8768bd2723425ed34256ff7b7a08eeb0bd373 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Feb 2018 21:21:28 +0100
Subject: bin/get-assignment: fix mysql parser to understand columns with
 spaces correctly

---
 bin/get-assignment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-assignment b/bin/get-assignment
index 3dbd386..e3299f6 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -218,7 +218,7 @@ next_building=$(
     sed '
       1d
       y/\t/ /
-      s/^\(\S\+ \)\{4\}//
+      s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/
     '
 )
 if [ -n "${next_building}" ]; then
-- 
cgit v1.2.3-54-g00ecf


From 426210589120997b4bf991636ea5a16acbb964ae Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 19 Feb 2018 08:58:35 +0100
Subject: bin/build-packages: remove old, residual tmpdirs

---
 bin/build-packages | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/bin/build-packages b/bin/build-packages
index b6c4a7a..b99036f 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -222,6 +222,11 @@ while [ "${count}" -ne 0 ]; do
       if [ "${mod_git_revision}" = 'work-tree' ]; then
         mod_git_revision=$(
           # we can't just create an empty index-file with mktemp, because git doesn't like it
+          find . \
+            -mindepth 1 \
+            -maxdepth 1 \
+            -name 'tmp.build-packages.git.*' \
+            -exec rm -rf --one-file-system {} \;
           tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir)
           trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT
           export GIT_INDEX_FILE="${tmp_subdir}/index.new"
@@ -255,6 +260,11 @@ while [ "${count}" -ne 0 ]; do
         recursively_umount_and_rm "${tmp_dir}"
         exit "${err}"
       }
+      find . \
+        -mindepth 1 \
+        -maxdepth 1 \
+        -name 'tmp.build-packages.??????' \
+        -exec rm -rf --one-file-system {} \;
       tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
       trap bail_out EXIT
 
-- 
cgit v1.2.3-54-g00ecf


From db88d4e5468084e37dbcf46361b04d22248e11fd Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 19 Feb 2018 19:18:32 +0100
Subject: bin/sanity-check: go insane if mysql database is not sane

---
 bin/sanity-check | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/bin/sanity-check b/bin/sanity-check
index 39a2d10..601a940 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -105,7 +105,7 @@ touch "${tmp_dir}/messages"
 trap 'finish' EXIT
 
 if [ $# -eq 0 ]; then
-  set -- git-repositories build-list repos package-database state-files
+  set -- git-repositories build-list mysql repos package-database state-files
 fi
 
 while [ $# -gt 0 ]; do
@@ -385,6 +385,26 @@ while [ $# -gt 0 ]; do
 
     ;;
 
+    mysql)
+
+      [ ${silence} -gt 0 ] || \
+        printf 'checking mysql-sanity-check-file ...' >> \
+        "${tmp_dir}/messages"
+
+      if [ -s "${webserver_directory}/mysql-sanity.html" ]; then
+        if [ ${silence} -le 1 ]; then
+          printf '\nThere is something wrong with the database:\n'
+          cat "${webserver_directory}/mysql-sanity.html"
+        fi
+        i_am_insane
+      fi
+
+      [ ${silence} -gt 0 ] || \
+        echo ' passed.' >> \
+        "${tmp_dir}/messages"
+
+    ;;
+
     *)
 
       [ ${silence} -gt 1 ] || \
-- 
cgit v1.2.3-54-g00ecf


From a4cb656d8fd5f7ca3358cc629ee2e8118f242089 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Feb 2018 06:51:46 +0100
Subject: bin/build-packages: add TODO

---
 bin/build-packages | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/build-packages b/bin/build-packages
index b99036f..0e734dd 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -307,6 +307,7 @@ while [ "${count}" -ne 0 ]; do
             # we can't improve anything
             continue
           fi
+          # TODO: the below command also overwrites files which are intentionally overwritten by us (e.g. "config" in "core/linux" or "extra/linux-zen")
           tar -xz --overwrite -f "${source_name}" --exclude PKGBUILD --strip-components=1 || true
         fi
 
-- 
cgit v1.2.3-54-g00ecf


From ff4a1f4cdb66d930fc781222a093b322f157e3f8 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 10:30:46 +0100
Subject: bin/build-packages: do not overwrite our files by upstream ones if
 source package is extracted

---
 bin/build-packages | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/bin/build-packages b/bin/build-packages
index 0e734dd..f15c99e 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -307,8 +307,19 @@ while [ "${count}" -ne 0 ]; do
             # we can't improve anything
             continue
           fi
-          # TODO: the below command also overwrites files which are intentionally overwritten by us (e.g. "config" in "core/linux" or "extra/linux-zen")
-          tar -xz --overwrite -f "${source_name}" --exclude PKGBUILD --strip-components=1 || true
+          # shellcheck disable=SC2046
+          tar -xz --overwrite \
+            -f "${source_name}" \
+            --exclude PKGBUILD \
+            $(
+              if [ -n "${PKGBUILD_mod}" ]; then
+                git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \
+                  tar -t | \
+                  sed 's/^/--exclude /'
+              fi
+            ) \
+            --strip-components=1 \
+            || true
         fi
 
         if echo "${straw}" | \
-- 
cgit v1.2.3-54-g00ecf


From fa16cc10d6ad6e6a43f9bd185fd17f1a18ae076e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 11:58:27 +0100
Subject: bin/return-assignment: add provided libraries into database

---
 bin/return-assignment | 84 +++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 72 insertions(+), 12 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 6f96ee0..72981bb 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -306,18 +306,33 @@ tar -x \
   --no-wildcards-match-slash \
   '*.pkg.tar.xz' \
   '*.pkg.tar.xz.sig' \
-  '*.pkg.tar.xz-namcap.log.gz'
-
-# check if all packages are signed and all signatures belong to a package
+  '*.pkg.tar.xz-namcap.log.gz' \
+  '*.pkg.tar.xz.so.provides'
+
+# check if all packages come with:
+#  - a package file
+#  - a signature
+#  - a namcap log
+#  - a list of provided libraries
 missing_files=$(
-  find . -maxdepth 1 -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' -o -name '*.pkg.tar.xz-namcap.log.gz' | \
-    sed '
-      s@\.sig$@ signature@
-      t
-      s@-namcap\.log\.gz$@ namcap@
-      t
-      s@$@ package@
-    ' | \
+  find . -maxdepth 1 \( \
+    \( \
+      -name '*.pkg.tar.xz' \
+      -printf '%f package\n' \
+    \) -o \
+    \( \
+      -name '*.pkg.tar.xz.sig' \
+      -printf '%f signature\n' \
+    \) -o \
+    \( \
+      -name '*.pkg.tar.xz-namcap.log' \
+      -printf '%f namcap\n' \
+    \) -o \
+    \( \
+      -name '*.pkg.tar.xz.so.provides.gz' \
+      -printf '%f provided-libraries\n' \
+    \) \
+  \) | \
     sort -k1,1 -k2,2 | \
     sed '
       :a
@@ -347,11 +362,17 @@ missing_files=$(
         p
         g
       }
+      / provided-libraries /!{
+        h
+        s/^\(\S\+\) .*$/List of by "\1" provided libraries is missing./
+        p
+        g
+      }
     '
 )
 
 if [ -n "${missing_files}" ]; then
-  >&2 echo 'The following packages lack a signature, namcap log or package file:'
+  >&2 echo 'The following packages lack a signature, namcap log, package file or list of provided libraries:'
   >&2 echo "${missing_files}"
   exit 3
 fi
@@ -423,6 +444,45 @@ fi
 # move namcap.logs
 find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_log_directory}/success/" \;
 
+# insert provided libraries into database
+# shellcheck disable=SC2016
+{
+  printf 'CREATE TEMPORARY TABLE `pl` (`pkgfile` VARCHAR(64), `lib` VARCHAR(64));\n'
+  find . -maxdepth 1 -name '*.pkg.tar.xz.so.provides.gz' -execdir zgrep -HF '' '{}' \; | \
+    sed -n '
+      s,^\./\(.\+\.pkg\.tar\.xz\)\.so\.provides\.gz:\([^:]\+\)$,\1\n\2,
+      T
+      p
+    ' | \
+    base64_encode_each | \
+    sed '
+      N
+      s/^\(\S\+\)\n\(\S\+\)$/(from_base64("\1"),from_base64("\2")),/
+      $s/,$/;/
+      2 s/^/INSERT INTO `pl` (`pkgfile`,`lib`) VALUES /
+    '
+  printf 'INSERT IGNORE INTO `install_targets` (`name`)'
+  printf ' SELECT DISTINCT `pl`.`lib` FROM `pl`;\n';
+  printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`)'
+  printf ' SELECT `binary_packages`.`id`,`install_targets`.`id`'
+  printf ' FROM `install_targets` JOIN `pl` ON `pl`.`lib`=`install_targets`.`name`'
+  printf ' JOIN `binary_packages`'
+  mysql_join_binary_packages_architectures
+  mysql_join_binary_packages_build_slaves
+  mysql_join_binary_packages_repositories
+  printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+    "$(
+      printf '%s' "${slave}" | \
+        base64 -w0
+    )"
+  printf ' AND `repositories`.`name`="build-list"'
+  printf ' AND '
+  mysql_package_name_query
+  printf '=`pl`.`pkgfile`'
+  printf ';\n'
+} | \
+  ${mysql_command}
+
 # move packages
 destination=$(official_or_community "$1.$2.$3.$4" 'staging')
 
-- 
cgit v1.2.3-54-g00ecf


From 9e612bcdf9fd521cde2873ea8c5368e05c1dbb46 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 12:31:34 +0100
Subject: bin/build-packages: save list of required libraries

---
 bin/build-packages | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index f15c99e..70a0c3a 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -426,7 +426,21 @@ while [ "${count}" -ne 0 ]; do
                 ' > \
                 "${pkgfile}.so.provides"
             done
-          # TODO: search for required libraries, too
+          >&2 printf 'searching for required libraries\n'
+          package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX")
+          find "${tar_content_dir}" -maxdepth 1 \
+            -name '*.pkg.tar.xz' | \
+            while read -r pkgfile; do
+              mkdir "${package_content_dir}/${pkgfile##*/}"
+              tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
+              find "${package_content_dir}/${pkgfile##*/}" -type f \
+                -exec objdump -x '{}' \; 2>/dev/null | \
+                grep -w 'NEEDED' | \
+                awk '{print $2}' | \
+                sort -u > \
+                "${pkgfile}.so.needs"
+              rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
+            done
           >&2 printf 'running namcap ...'
           if [ "${repository}" = 'multilib' ]; then
             x86_64_build_command='multilib-build'
@@ -467,6 +481,7 @@ while [ "${count}" -ne 0 ]; do
             find "${tar_content_dir}/" -maxdepth 1 \
               \( \
                 -name '*.pkg.tar.xz-namcap.log' -o \
+                -name '*.pkg.tar.xz.so.needs' -o \
                 -name '*.pkg.tar.xz.so.provides' \
               \) \
               -execdir gzip '{}' \;
@@ -482,6 +497,7 @@ while [ "${count}" -ne 0 ]; do
                 -name '*.pkg.tar.xz' -o \
                 -name '*.pkg.tar.xz.sig' -o \
                 -name '*.pkg.tar.xz-namcap.log.gz' -o \
+                -name '*.pkg.tar.xz.so.needs.gz' -o \
                 -name '*.pkg.tar.xz.so.provides.gz' \
               \) \
               -printf '%f\n'
-- 
cgit v1.2.3-54-g00ecf


From a7daeeb2c73cdf2eaa3377ce972b5ec17a8e47a1 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 13:53:45 +0100
Subject: bin/return-assignment: bugfix :-/

---
 bin/return-assignment | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 72981bb..ab4f033 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -307,7 +307,7 @@ tar -x \
   '*.pkg.tar.xz' \
   '*.pkg.tar.xz.sig' \
   '*.pkg.tar.xz-namcap.log.gz' \
-  '*.pkg.tar.xz.so.provides'
+  '*.pkg.tar.xz.so.provides.gz'
 
 # check if all packages come with:
 #  - a package file
@@ -325,7 +325,7 @@ missing_files=$(
       -printf '%f signature\n' \
     \) -o \
     \( \
-      -name '*.pkg.tar.xz-namcap.log' \
+      -name '*.pkg.tar.xz-namcap.log.gz' \
       -printf '%f namcap\n' \
     \) -o \
     \( \
@@ -333,7 +333,10 @@ missing_files=$(
       -printf '%f provided-libraries\n' \
     \) \
   \) | \
-    sort -k1,1 -k2,2 | \
+    sed '
+      s/\(\.pkg\.tar\.xz\)\(\.sig\|-namcap\.log\.gz\|\.so\.provides\.gz\) /\1 /
+    ' | \
+    sort -k1,1 -k2,2 | tee /dev/stderr | \
     sed '
       :a
         $!N
@@ -364,7 +367,7 @@ missing_files=$(
       }
       / provided-libraries /!{
         h
-        s/^\(\S\+\) .*$/List of by "\1" provided libraries is missing./
+        s/^\(\S\+\) .*$/List of libraries provided by "\1" is missing./
         p
         g
       }
-- 
cgit v1.2.3-54-g00ecf


From ab2b33c9cf933ad5e023ae4b2c79497f11c12c00 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 14:21:51 +0100
Subject: bin/return-assignment: require list of needed libraries

---
 bin/return-assignment | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index ab4f033..2b3a643 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -307,12 +307,14 @@ tar -x \
   '*.pkg.tar.xz' \
   '*.pkg.tar.xz.sig' \
   '*.pkg.tar.xz-namcap.log.gz' \
+  '*.pkg.tar.xz.so.needs.gz' \
   '*.pkg.tar.xz.so.provides.gz'
 
 # check if all packages come with:
 #  - a package file
 #  - a signature
 #  - a namcap log
+#  - a list of needed libraries
 #  - a list of provided libraries
 missing_files=$(
   find . -maxdepth 1 \( \
@@ -328,13 +330,17 @@ missing_files=$(
       -name '*.pkg.tar.xz-namcap.log.gz' \
       -printf '%f namcap\n' \
     \) -o \
+    \( \
+      -name '*.pkg.tar.xz.so.needs.gz' \
+      -printf '%f needed-libraries\n' \
+    \) -o \
     \( \
       -name '*.pkg.tar.xz.so.provides.gz' \
       -printf '%f provided-libraries\n' \
     \) \
   \) | \
     sed '
-      s/\(\.pkg\.tar\.xz\)\(\.sig\|-namcap\.log\.gz\|\.so\.provides\.gz\) /\1 /
+      s/\(\.pkg\.tar\.xz\)\(\.sig\|-namcap\.log\.gz\|\.so\.\(provides\|needs\)\.gz\) /\1 /
     ' | \
     sort -k1,1 -k2,2 | tee /dev/stderr | \
     sed '
@@ -365,6 +371,12 @@ missing_files=$(
         p
         g
       }
+      / needed-libraries /!{
+        h
+        s/^\(\S\+\) .*$/List of libraries needed by "\1" is missing./
+        p
+        g
+      }
       / provided-libraries /!{
         h
         s/^\(\S\+\) .*$/List of libraries provided by "\1" is missing./
@@ -375,7 +387,7 @@ missing_files=$(
 )
 
 if [ -n "${missing_files}" ]; then
-  >&2 echo 'The following packages lack a signature, namcap log, package file or list of provided libraries:'
+  >&2 echo 'The following packages lack a signature, namcap log, package file or list of needed/provided libraries:'
   >&2 echo "${missing_files}"
   exit 3
 fi
-- 
cgit v1.2.3-54-g00ecf


From 9f56bebe359b46849c3893601953cc21d186c207 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 14:52:03 +0100
Subject: bin/return-assignment: add TODO

---
 bin/return-assignment | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/return-assignment b/bin/return-assignment
index 2b3a643..8ce0de6 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -476,6 +476,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_
       $s/,$/;/
       2 s/^/INSERT INTO `pl` (`pkgfile`,`lib`) VALUES /
     '
+  # TODO: insert needed libraries into database, too
   printf 'INSERT IGNORE INTO `install_targets` (`name`)'
   printf ' SELECT DISTINCT `pl`.`lib` FROM `pl`;\n';
   printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`)'
-- 
cgit v1.2.3-54-g00ecf


From b0221d5ad5df2e4b06a194b0833d2fc7a5c7b119 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 14:53:25 +0100
Subject: bin/build-master-status-from-mysql: make loop-error non-fatal

---
 bin/build-master-status-from-mysql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index ea8d0f1..b55ca65 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -244,7 +244,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     true
   diff -u \
     "${tmp_dir}/loops.file" \
-    "${tmp_dir}/loops.mysql" || \
+    "${tmp_dir}/loops.mysql" >&2 || \
     true
   diff -u \
     "${tmp_dir}/tested.file" \
-- 
cgit v1.2.3-54-g00ecf


From ae61aa2cc45938cfd467909c97392b0998cd6cad Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 17:12:02 +0100
Subject: bin/return-assignment: remove debug output - whoops

---
 bin/return-assignment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 8ce0de6..c945dbb 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -342,7 +342,7 @@ missing_files=$(
     sed '
       s/\(\.pkg\.tar\.xz\)\(\.sig\|-namcap\.log\.gz\|\.so\.\(provides\|needs\)\.gz\) /\1 /
     ' | \
-    sort -k1,1 -k2,2 | tee /dev/stderr | \
+    sort -k1,1 -k2,2 | \
     sed '
       :a
         $!N
-- 
cgit v1.2.3-54-g00ecf


From da479ecca22e3b49fb379ac8e5b77c6a9ac1376f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 17:26:05 +0100
Subject: bin/seed-build-list: print seeded packages to stdout, too

---
 bin/seed-build-list | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index aff7acb..284c555 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -301,6 +301,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     flock -s 8
 
     while read -r pkgbase git_rev mod_git_rev repo; do
+      printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
       generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
-- 
cgit v1.2.3-54-g00ecf


From 74fd125129a7585484ce455d7fd55da5a930691b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Feb 2018 17:46:10 +0100
Subject: bin/return-assignment: save needed libraries in database

---
 bin/return-assignment | 84 ++++++++++++++++++++++++++++++---------------------
 1 file changed, 50 insertions(+), 34 deletions(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index c945dbb..db433ee 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -462,40 +462,56 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_
 # insert provided libraries into database
 # shellcheck disable=SC2016
 {
-  printf 'CREATE TEMPORARY TABLE `pl` (`pkgfile` VARCHAR(64), `lib` VARCHAR(64));\n'
-  find . -maxdepth 1 -name '*.pkg.tar.xz.so.provides.gz' -execdir zgrep -HF '' '{}' \; | \
-    sed -n '
-      s,^\./\(.\+\.pkg\.tar\.xz\)\.so\.provides\.gz:\([^:]\+\)$,\1\n\2,
-      T
-      p
-    ' | \
-    base64_encode_each | \
-    sed '
-      N
-      s/^\(\S\+\)\n\(\S\+\)$/(from_base64("\1"),from_base64("\2")),/
-      $s/,$/;/
-      2 s/^/INSERT INTO `pl` (`pkgfile`,`lib`) VALUES /
-    '
-  # TODO: insert needed libraries into database, too
-  printf 'INSERT IGNORE INTO `install_targets` (`name`)'
-  printf ' SELECT DISTINCT `pl`.`lib` FROM `pl`;\n';
-  printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`)'
-  printf ' SELECT `binary_packages`.`id`,`install_targets`.`id`'
-  printf ' FROM `install_targets` JOIN `pl` ON `pl`.`lib`=`install_targets`.`name`'
-  printf ' JOIN `binary_packages`'
-  mysql_join_binary_packages_architectures
-  mysql_join_binary_packages_build_slaves
-  mysql_join_binary_packages_repositories
-  printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
-    "$(
-      printf '%s' "${slave}" | \
-        base64 -w0
-    )"
-  printf ' AND `repositories`.`name`="build-list"'
-  printf ' AND '
-  mysql_package_name_query
-  printf '=`pl`.`pkgfile`'
-  printf ';\n'
+  for lib_link in 'pl:provides' 'nl:needs'; do
+    printf 'CREATE TEMPORARY TABLE `%s` (`pkgfile` VARCHAR(64), `lib` VARCHAR(64));\n' \
+      "${lib_link%:*}"
+    find . -maxdepth 1 -name '*.pkg.tar.xz.so.'"${lib_link#*:}"'.gz' -execdir zgrep -HF '' '{}' \; | \
+      sed -n '
+        s,^\./\(.\+\.pkg\.tar\.xz\)\.so\.'"${lib_link#*:}"'\.gz:\([^:]\+\)$,\1\n\2,
+        T
+        p
+      ' | \
+      base64_encode_each | \
+      sed '
+        N
+        s/^\(\S\+\)\n\(\S\+\)$/(from_base64("\1"),from_base64("\2")),/
+        $s/,$/;/
+        2 s/^/INSERT INTO `'"${lib_link%:*}"'` (`pkgfile`,`lib`) VALUES /
+      '
+    printf 'INSERT IGNORE INTO `install_targets` (`name`)'
+    printf ' SELECT DISTINCT `%s`.`lib` FROM `%s`;\n' \
+      "${lib_link%:*}" "${lib_link%:*}"
+    if [ "${lib_link%:*}" = 'pl' ]; then
+      printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`)'
+    else
+      printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`)'
+    fi
+    printf ' SELECT `binary_packages`.`id`,`install_targets`.`id`'
+    if [ "${lib_link%:*}" = 'nl' ]; then
+      printf ',`dependency_types`.`id`'
+    fi
+    printf ' FROM `install_targets`'
+    if [ "${lib_link%:*}" = 'nl' ]; then
+      printf ' JOIN `dependency_types` ON `dependency_types`.`name`="link"'
+    fi
+    printf ' JOIN `%s` ON `%s`.`lib`=`install_targets`.`name`' \
+      "${lib_link%:*}" "${lib_link%:*}"
+    printf ' JOIN `binary_packages`'
+    mysql_join_binary_packages_architectures
+    mysql_join_binary_packages_build_slaves
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+      "$(
+        printf '%s' "${slave}" | \
+          base64 -w0
+      )"
+    printf ' AND `repositories`.`name`="build-list"'
+    printf ' AND '
+    mysql_package_name_query
+    printf '=`%s`.`pkgfile`' \
+      "${lib_link%:*}"
+    printf ';\n'
+  done
 } | \
   ${mysql_command}
 
-- 
cgit v1.2.3-54-g00ecf


From 2e6d97b8ddcd16f029673bc6f101fc3baf141e51 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 24 Feb 2018 19:41:56 +0100
Subject: bin/common-functions -> lib/common-functions, bin/mysql-functions ->
 lib/mysql-functions

---
 bin/common-functions | 1378 --------------------------------------------------
 bin/mysql-functions  | 1098 ----------------------------------------
 conf/default.conf    |    8 +-
 lib/common-functions | 1378 ++++++++++++++++++++++++++++++++++++++++++++++++++
 lib/mysql-functions  | 1098 ++++++++++++++++++++++++++++++++++++++++
 5 files changed, 2480 insertions(+), 2480 deletions(-)
 delete mode 100755 bin/common-functions
 delete mode 100755 bin/mysql-functions
 create mode 100755 lib/common-functions
 create mode 100755 lib/mysql-functions

diff --git a/bin/common-functions b/bin/common-functions
deleted file mode 100755
index a84271b..0000000
--- a/bin/common-functions
+++ /dev/null
@@ -1,1378 +0,0 @@
-#!/bin/sh
-
-# contains functions used by more than one script
-
-# shellcheck disable=SC2039
-
-# TODO: include link depenendencies in run-depends metadata
-
-# TODO: have full information (currently in files) in database
-
-# TODO: remove state files / metadata files
-
-if [ -z "${base_dir}" ]; then
-  # just to make shellcheck happy
-  . 'conf/default.conf'
-fi
-
-# find_pkgbuilds package repository git_repository git_revision mod_git_revision
-# find the PKGBUILD and modification of $package from $repository
-# sets $PKGBUILD and $PKGBUILD_mod
-
-find_pkgbuilds() {
-
-  local package="$1"
-  local repository="$2"
-  local git_repository="$3"
-  local git_revision="$4"
-  local mod_git_revision="$5"
-
-  local repo_path
-  eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
-  PKGBUILD=$(
-    git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \
-      tar -t 2> /dev/null | \
-      grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \
-      grep -v -- '-i686/PKGBUILD$' | \
-      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
-      sort | \
-      tail -n1
-  )
-
-  PKGBUILD_mod=$(
-    git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \
-      tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null
-  ) || true
-
-  if [ -z "${PKGBUILD}" ] && \
-    [ -z "${PKGBUILD_mod}" ]; then
-    >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \
-      "${package}" \
-      "${repository}" \
-      "${git_repository}" \
-      "${git_revision}" \
-      "${mod_git_revision}"
-    return 1
-  fi
-
-}
-
-# find_repository_with_commit commit
-# find the repository which has $commit
-
-find_repository_with_commit() {
-
-  local repository
-
-  for repository in ${repo_names}; do
-    # shellcheck disable=SC2016
-    if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then
-      echo "${repository}"
-      return 0
-    fi
-  done
-  >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1"
-  exit 1
-
-}
-
-# find_git_repository_to_package_repository repository
-# find the git repository which tracks the package repository $repository
-
-find_git_repository_to_package_repository() {
-
-  local repository
-  local package_repository
-  local repo_path
-
-  package_repository="$1"
-
-  if [ "$1" = 'build-support' ]; then
-    echo 'packages'
-    return 0
-  fi
-
-  for repository in ${repo_names}; do
-    if [ "${repository}" = "archlinux32" ]; then
-      continue
-    fi
-    eval 'repo_path="${repo_paths__'"${repository}"'}"'
-    if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \
-      tar -t --wildcards '*/repos' | \
-      grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \
-      cut -d/ -f3 | \
-      sed 's|-[^-]\+$||' | \
-      sort -u | \
-      grep -qxF "${package_repository}"; then
-      echo "${repository}"
-      return 0
-    fi
-  done
-  >&2 echo "can't find git repository with package repository '$1'"
-  exit 1
-
-}
-
-# generate_package_metadata $package $git_revision $mod_git_revision $repository
-# or
-# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# generate the meta data files of a package (dependencies, built packages, ...)
-
-generate_package_metadata() {
-
-  local package="$1"
-  local git_revision="$2"
-  local mod_git_revision="$3"
-  local repository="$4"
-  local file_prefix
-  local file
-  local PKGBUILD
-
-  if [ $# -eq 1 ]; then
-    # second form
-    repository="${package##*.}"
-    package="${package%.*}"
-    mod_git_revision="${package##*.}"
-    package="${package%.*}"
-    git_revision="${package##*.}"
-    package="${package%.*}"
-  fi
-
-  file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
-
-  if [ -e "${file_prefix}.builds" ] && \
-    [ -e "${file_prefix}.build-depends" ] && \
-    [ -e "${file_prefix}.run-depends" ] && \
-    [ -e "${file_prefix}.groups" ] && \
-    [ -e "${file_prefix}.packages" ]; then
-    return 0
-  fi
-
-  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then
-    printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-  if [ ! -s "${file_prefix}.SRCINFO" ]; then
-    >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-
-  # otherwise this just calls for trouble
-  sed -i '
-    /^[^=]*=\s*$/d
-    s/_i686\(\s*=\)/\1/
-  ' "${file_prefix}.SRCINFO"
-
-  # extract "groups" = groups \cup provides
-  grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.groups"
-
-  # extract "packages" = pkgname
-  grep '^pkgname = ' "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.packages"
-
-  # extract "builds" = provides \cup pkgname \cup groups
-  cat "${file_prefix}.groups" "${file_prefix}.packages" | \
-    sort -u > \
-    "${file_prefix}.builds"
-
-  # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base' 'base-devel'
-      sed -n "$(
-        printf '/^\t%s = /p\n' \
-          'depends' \
-          'makedepends'
-      )" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.build-depends"
-
-  # extract "run-depends" = depends \cup \{ base \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base'
-      sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.run-depends"
-
-  rm "${file_prefix}.SRCINFO"
-
-}
-
-# delete_old_metadata
-# delete old (=unneeded) meta data of packages
-
-delete_old_metadata() {
-
-  local current_metadata
-
-  current_metadata=$(
-    find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.[^.]\+$||
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
-      ' | \
-      sort -u
-  )
-
-  ( # the new shell is intentional
-    # what we have
-    echo "${current_metadata}"
-
-    # package-states should stay
-    find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
-      ' | \
-      sort -u | \
-      sed 'p'
-
-    # build-list items should stay
-    sed 'p' "${work_dir}/build-list"
-
-    tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    echo "${current_metadata}" | \
-      sort -k1,1 > \
-      "${tmp_dir}/current-metadata"
-
-    # the newest of the following should stay:
-    {
-      # deletion-list items
-      cat "${work_dir}/deletion-list"
-      # all packages in the repos
-      for repo in ${repo_names}; do
-        eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \
-          tar -t | \
-          sed '
-            s|/$||
-            /\//d
-          '
-      done
-    } | \
-      sort -u | \
-      join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
-      sort -k4,4 > \
-      "${tmp_dir}/find-newest-revisions"
-
-    uniq -uf3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-
-    uniq -Df3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      uniq --group=append -f3 | \
-      {
-        revs=''
-        mod_revs=''
-        opkg=''
-        orepo=''
-        while read -r rev mod_rev repo pkg; do
-
-          if [ -z "${rev}" ] && \
-            [ -z "${mod_rev}" ] && \
-            [ -z "${repo}" ] && \
-            [ -z "${pkg}" ]; then
-
-            printf '%s %s %s %s\n' \
-              "$(
-                printf '%s\n' ${revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "$(
-                printf '%s\n' ${mod_revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "${orepo}" \
-              "${opkg}"
-
-            revs=''
-            mod_revs=''
-            orepo=''
-            opkg=''
-            continue
-          fi
-          revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${revs} ${rev} | \
-              sort -u
-          )
-          mod_revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${mod_revs} ${mod_rev} | \
-              sort -u
-          )
-          orepo="${repo}"
-          opkg="${pkg}"
-        done
-      } | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-  ) | \
-    sort | \
-    uniq -u | \
-    while read -r pkg rev mod_rev repo; do
-      rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
-    done
-}
-
-# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
-# print which (stable) repository a package belongs to
-
-repository_of_package() {
-  local package="$1"
-  local repository="${package##*.}"
-  package="${package%.*}"
-  local a32_rev="${package##*.}"
-  package="${package%.*.*}"
-
-  case "${repository}" in
-    'multilib')
-      if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \
-        tar -Ox | \
-        grep -qFx "${package%.*.*.*}"; then
-        echo 'extra'
-      else
-        echo 'community'
-      fi
-      ;;
-    *)
-      echo "${repository}"
-  esac
-}
-
-# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending
-# print wether the specified package is an official package (print
-# $ending) or a community package (print 'community-$ending') or a
-# build-support package (print 'build-support')
-
-official_or_community() {
-  local prepo
-  prepo=$(repository_of_package "$1")
-
-  if [ "${prepo}" = 'community' ]; then
-    echo 'community-'"$2"
-  elif [ "${prepo}" = 'build-support' ]; then
-    echo 'build-support'
-  else
-    echo "$2"
-  fi
-}
-
-# ls_master_mirror $path
-# list content of $path on the master mirror (via rsync)
-
-ls_master_mirror() {
-
-  local path="$1"
-
-  ${master_mirror_rsync_command} \
-    "${master_mirror_rsync_directory}/${path}/" | \
-    grep -v '\s\.$' | \
-    awk '{print $5}'
-
-}
-
-# TODO: the actions of remove_old_package_versions should be done
-# on basis of the information in the database
-
-# remove_old_package_versions $arch $repository $package_file
-
-# removes all older (not-newer) versions of $package_file
-# in all repositories not-older (newer) than $repository
-
-# TODO: should remove all other version (also newer) from
-# some repositories :-/
-
-# A package is considered not newer if
-#  a) its version is not newer
-# A package is considered older if
-#  b) its version is older or
-#  c) if it's "not newer" and its architecture is 'any' and different or
-#  d) if it's "not newer" and the other architecture is 'any' and different
-
-# this ensures an any package may replace arch-specific packages of the same version and vice versa
-
-remove_old_package_versions() {
-
-  local arch="$1"
-  local repository="$2"
-  local package="$3"
-
-  local pkgname
-  local epoch
-  local pkgver
-  local pkgrel
-  local sub_pkgrel
-  pkgname="${package%-*}"
-  pkgrel="${pkgname##*-}"
-  sub_pkgrel="${pkgrel##*.}"
-  if [ "${sub_pkgrel}" = "${pkgrel}" ]; then
-    sub_pkgrel='0'
-  else
-    pkgrel="${pkgrel%.*}"
-  fi
-  pkgname="${pkgname%-*}"
-  pkgver="${pkgname##*-}"
-  epoch="${pkgver%%:*}"
-  if [ "${epoch}" = "${pkgver}" ]; then
-    epoch='0'
-  else
-    pkgver="${pkgver#*:}"
-  fi
-  pkgname="${pkgname%-*}"
-
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \
-      "$(
-        printf '%s' "${repository}" | \
-          base64 -w0
-      )" \
-      "$(
-        printf '%s' "${package}" | \
-          base64 -w0
-      )" \
-      "$(
-        printf '%s' "${package}" | \
-          sed '
-            s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/
-          ' | \
-          base64 -w0
-      )"
-    printf 'SELECT '
-    printf '`binary_packages`.`id`,'
-    printf 'CONCAT(`repositories`.`name`,"/",'
-    mysql_package_name_query
-    printf '),'
-    # should we delete packages of identical version?
-    printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),'
-    printf 'CONCAT('
-      printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
-      printf '`binary_packages`.`pkgver`,"-",'
-      printf '`binary_packages`.`pkgrel`,".",'
-      printf '`binary_packages`.`sub_pkgrel`'
-    printf ')'
-    printf ' FROM `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_binary_packages_architectures
-    printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
-    printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`'
-    # name must match
-    printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \
-      "$(printf '%s' "${package%-*-*-*}" | base64 -w0)"
-    # repository, where package should be deleted, should be less stable
-    printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \
-      "$(printf '%s' "${repository}" | base64 -w0)"
-    printf ';\n'
-  } | \
-    ${mysql_command} --raw --batch | \
-    sed '
-      /^\S\+\sCONCAT(/d
-      y/\t/ /
-    ' | \
-    expand_version 4 | \
-    sort -k4V,4 -k3r,3 | \
-    shrink_version 4 | \
-    sed -n '
-      /^bogus /q
-      p
-    ' | \
-    cut -d' ' -f1,2 >&2
-
-  # repositories in which older packages should be deleted
-  local delete_older_repositories
-  # repositories in which not-newer packages should be deleted
-  local delete_not_newer_repositories
-
-  if echo "${standalone_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories="${repository}"
-    delete_not_newer_repositories=''
-
-  elif echo "${staging_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories="${repository}"
-    delete_not_newer_repositories=$(
-      echo "${staging_package_repositories}" | \
-        grep -vxF "${repository}"
-    ) || true
-
-  elif echo "${testing_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories=$(
-      printf '%s\n' "${staging_package_repositories}" "${repository}"
-    )
-    delete_not_newer_repositories=$(
-      echo "${testing_package_repositories}" | \
-        grep -vxF "${repository}"
-    ) || true
-
-  elif echo "${stable_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories=$(
-      printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}"
-    )
-    delete_not_newer_repositories=$(
-      echo "${stable_package_repositories}" | \
-        grep -vxF "${repository}"
-    ) || true
-
-  else
-
-    >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}"
-    return 1
-
-  fi
-
-  ( # the new shell is intentional
-    tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    {
-      # the architecture of the package (any vs. i686)
-      package_arch="${package##*-}"
-      package_arch="${package_arch%%.*}"
-      if [ "${package_arch}" = 'any' ]; then
-        package_arch_regex_inverter='!'
-      else
-        unset package_arch_regex_inverter
-      fi
-
-      for repo in ${delete_older_repositories}; do
-        ls_master_mirror "${arch}/${repo}" | \
-          sed -n '
-            /\.pkg\.tar\.xz$/!d
-            s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
-            /^'"$(str_to_regex "${package%-*-*-*}")"' / {
-              s|^|2 '"${arch} ${repo}"' |
-              / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{
-                s|^2|0|
-              }
-              p
-            }
-          '
-      done
-      for repo in ${delete_not_newer_repositories}; do
-        ls_master_mirror "${arch}/${repo}" | \
-          sed -n '
-            /\.pkg\.tar\.xz$/!d
-            s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
-            /^'"$(str_to_regex "${package%-*-*-*}")"' / {
-              s|^|0 '"${arch} ${repo}"' |
-              p
-            }
-          '
-      done
-      echo "${package%-*}" | \
-        sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|'
-
-      # the generated list contains the following columns:
-      # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz
-    } | \
-      expand_version 5 | \
-      sort -k5V,5 -k1n,1 | \
-      shrink_version 5 | \
-      sed -n '
-        /^1 %cut% %it% %here% /q
-        s/^[02] //
-        s/ \(\S\+\)$/-\1/
-        p
-      ' | \
-      sort -u > \
-      "${tmp_dir}/packages-to-delete"
-    # this file contains a list of packages to be deleted, one on each line:
-    # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz
-
-    cut -d' ' -f1,2 < \
-      "${tmp_dir}/packages-to-delete" | \
-      grep -vxF "${arch} ${repository}" | \
-      sort -u > \
-      "${tmp_dir}/repositories-to-modify"
-
-    # fetch all databases being modified
-    while read -r del_arch del_repo; do
-      mkdir -p "${tmp_dir}/${del_arch}/${del_repo}"
-      ${master_mirror_rsync_command} \
-        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \
-        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \
-        "${tmp_dir}/${del_arch}/${del_repo}/"
-    done < \
-      "${tmp_dir}/repositories-to-modify"
-
-    while read -r del_arch del_repo del_package _; do
-      if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then
-        # we do not repo-remove the package in the target repository
-        continue
-      fi
-      repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \
-        "${del_package}"
-    done < \
-      "${tmp_dir}/packages-to-delete"
-
-    # upload modified databases
-    while read -r del_arch del_repo; do
-      ${master_mirror_rsync_command} \
-        "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \
-        "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \
-        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/"
-    done < \
-      "${tmp_dir}/repositories-to-modify"
-
-    # shellcheck disable=SC2016
-    sed '
-      s/\.pkg\.tar\.xz$//
-      s/^\S\+ //
-      s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/
-      s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/
-      s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/
-    ' "${tmp_dir}/packages-to-delete" | \
-      while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do
-        printf 'DELETE FROM `binary_packages` WHERE'
-        printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \
-          'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \
-          'repository'   'repositories'  'repositories'  'repositories'  "$(printf '%s' "${repo}" | base64 -w0)"
-        printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-          'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \
-          'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
-          'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
-          'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
-          'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \
-          sed 's/ AND$//'
-        printf ';\n'
-      done | \
-      ${mysql_command}
-
-    sed '
-      s| \(\S\+\)$|-\1|
-      y| |/|
-      s|^|rm "|
-      s|$|"|
-      p
-      s|"$|.sig"|
-    ' "${tmp_dir}/packages-to-delete" | \
-      ${master_mirror_sftp_command}
-  )
-
-}
-
-# wait_some_time $minimum $diff
-# wait between minimum and minimum+diff seconds (diff defaults to 30)
-
-wait_some_time() {
-  local minimum=$1
-  local diff=$2
-  local random
-
-  if [ -z "${diff}" ]; then
-    diff=30
-  fi
-
-  random=$(
-    dd if='/dev/urandom' count=1 2> /dev/null | \
-      cksum | \
-      cut -d' ' -f1
-  )
-
-  sleep $((minimum + random % diff))
-}
-
-# str_to_regex $string
-# escape dots for use in regex
-
-str_to_regex() {
-  echo "$1" | \
-    sed '
-      s|[.[]|\\\0|g
-    '
-}
-
-# make_source_info $package $repository $git_revision $mod_git_revision $output
-# create .SRCINFO from PKGBUILD within git repositories, output to $output
-
-make_source_info() {
-
-  local package="$1"
-  local repository="$2"
-  local git_revision="$3"
-  local mod_git_revision="$4"
-  local output="$5"
-
-  local git_repo
-  local PKGBUILD
-  local PKGBUILD_mod
-
-  git_repo=$(find_repository_with_commit "${git_revision}")
-
-  if [ -z "${git_repo}" ]; then
-    return 1
-  fi
-
-  find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
-
-  ( # the new shell is intentional
-
-    tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX")
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0'
-
-    {
-      cd "${tmp_dir}"
-      makepkg --printsrcinfo
-      cd ..
-    } |
-    if [ "${package%-i18n}-i18n" = "${package}" ]; then
-      sed '
-        1 a \\tdepends = '"${package%-i18n}"'
-      '
-    else
-      cat
-    fi > \
-      "${output}"
-
-  )
-
-}
-
-# recursively_umount_and_rm $dir
-# umount all mountpoints in $dir which are also in $dir's
-#  filesystem, possibly also $dir itself and then
-#  rm -rf --one-file-system $dir
-
-recursively_umount_and_rm() {
-  local dir="$1"
-
-  if [ -z "${dir}" ]; then
-    >&2 echo 'ERROR: recursively_umount_and_rm requires an argument'
-    exit 42
-  fi
-
-  find "${dir}" \
-    -xdev -depth -type d \
-    -exec 'mountpoint' '-q' '{}' ';' \
-    -exec 'sudo' 'umount' '-l' '{}' ';'
-  rm -rf --one-file-system "${dir}"
-}
-
-# mangle_pkgbuild $PKGBUILD [$sub_pkgrel]
-#  mangle $arch in PKBUILDs to contain i486, i586, i686
-#  append $sub_pkgrel to the pkgrel
-
-mangle_pkgbuild() {
-  local PKGBUILD="$1"
-  local sub_pkgrel="$2"
-
-  if [ -n "${sub_pkgrel}" ]; then
-    sub_pkgrel=".${sub_pkgrel}"
-  fi
-
-  sed -i '
-    /^arch=[^#]*any/!{
-      /^arch=(/s/(/(i686 /
-    }
-    s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/
-  ' "${PKGBUILD}"
-}
-
-# find_newest_of_git_revisions
-# find newest git revision of the ones provided at stdin
-# (assuming linear history)
-
-find_newest_of_git_revisions() {
-  local revisions
-  local repo
-  revisions=$(cat)
-
-  if [ "$(
-    echo "${revisions}" | \
-      wc -l
-    )" -eq 1 ]; then
-
-    echo "${revisions}"
-    return
-
-  fi
-
-  repo=$(
-    find_repository_with_commit \
-      "$(
-        echo "${revisions}" | \
-          grep -xm1 '[0-9a-f]\{40\}'
-      )"
-  )
-
-  eval 'repo="${repo_paths__'"${repo}"'}"'
-
-  echo "${revisions}" | \
-    xargs -rn1 git -C "${repo}" rev-parse | \
-    {
-      newest=''
-      while read -r current; do
-        if [ -z "${newest}" ] || \
-          git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then
-            newest="${current}"
-          fi
-      done
-      echo "${newest}"
-    }
-}
-
-# find_package_repository_to_package $package $git_repository $git_commit
-# find the package repository a package from a given git repository
-# belongs to
-
-find_package_repository_to_package() {
-
-  local package="$1"
-  local git_repository="$2"
-  local git_commit="$3"
-  local repo_path
-  local repo
-
-  eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
-  if [ "${git_repository}" = 'archlinux32' ]; then
-    repo=$(
-      git -C "${repo_path}" archive "${git_commit}" -- | \
-        tar -t --wildcards "*/${package}/" | \
-        cut -d/ -f1 | \
-        sort -u
-    )
-  else
-    repo=$(
-      git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \
-        tar -t | \
-        cut -d/ -f3 | \
-        grep -vxF '' | \
-        grep -v 'staging\|testing\|-unstable' | \
-        grep -v -- '-i686$' | \
-        sed 's|-[^-]\+$||' | \
-        sort -u
-    )
-  fi
-
-  if [ -z "${repo}" ]; then
-    return 1
-  fi
-
-  if [ "$(
-    echo "${repo}" | \
-      wc -l
-    )" -ne 1 ]; then
-    return 1
-  fi
-
-  echo "${repo}"
-
-}
-
-# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel
-# extract files found in the svn/git source directories
-# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly
-
-extract_source_directory() {
-
-  local git_repo="$1"
-  local rev="$2"
-  local mod_rev="$3"
-  local output="$4"
-  local sub_pkgrel="$5"
-
-  if [ -n "${PKGBUILD}" ]; then
-    eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \
-      tar -x --strip-components=3 -C "${output}"
-  fi
-
-  if [ -n "${PKGBUILD_mod}" ]; then
-    git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \
-      tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \
-      true
-    git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \
-      tar -Ox "${PKGBUILD_mod}" >> \
-      "${output}/PKGBUILD"
-  fi
-
-  mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}"
-
-  # shellcheck disable=SC2016
-  sed -i '/^\$Id\$$/d' "${output}/PKGBUILD"
-
-}
-
-# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository
-# return a list of packages on the build list which are (run- / build- / check-time)
-# dependencies of the given package
-
-find_dependencies_on_build_list() {
-
-  local package="$1"
-  local git_revision="$2"
-  local mod_git_revision="$3"
-  local repository="$4"
-
-  generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
-  {
-    cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
-    awk '{print $1 "." $2 "." $3 "." $4}' < \
-      "${work_dir}/build-list" | \
-      sed '
-        s|^|'"${work_dir}"'/package-infos/|
-        s|$|\.builds|
-      ' | \
-      xargs -r cat | \
-      sort -u
-  } | \
-    sort | \
-    uniq -d
-
-}
-
-# download_sources_by_hash $package $repository $git_revision $git_mod_revision
-# try to download all sources by their hash into the current directory
-# returns 0 if any source was downloaded and 1 otherwise
-
-download_sources_by_hash() {
-
-  local package="$1"
-  local repository="$2"
-  local git_revision="$3"
-  local git_mod_revision="$4"
-
-  local return_value=1
-  local tmp_dir
-  local sum_type
-  local arch_suffix
-
-  tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir)
-
-  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then
-    >&2 echo 'download_sources_by_hash: make_source_info failed.'
-    rm -rf --one-file-system "${tmp_dir}"
-    return 1
-  fi
-
-  if ! [ -s "${tmp_dir}/.SRCINFO" ]; then
-    >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.'
-    rm -rf --one-file-system "${tmp_dir}"
-    return 1
-  fi
-
-  for arch_suffix in '' '_i686'; do
-    for sum_type in 'sha256sum' 'sha512sum'; do
-      grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
-        sed 's|^.* = ||' | \
-        cat -n > \
-        "${tmp_dir}/sums"
-      grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
-        sed '
-          s|^.* = ||
-          s|::.*$||
-          s|.*/||
-        ' | \
-        cat -n > \
-        "${tmp_dir}/urls"
-      if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then
-        join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \
-          "${tmp_dir}/joined"
-        while read -r sum file; do
-          if [ "${sum}" = 'SKIP' ]; then
-            continue
-          fi
-          if echo "${sum}  ${file}" | \
-            ${sum_type} -c > /dev/null 2>&1; then
-            # the correct source is already there
-            continue
-          fi
-          if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then
-            mv "${tmp_dir}/transfer" "${file}"
-            return_value=0
-          fi
-        done < \
-          "${tmp_dir}/joined"
-      fi
-    done
-  done
-
-  rm -rf --one-file-system "${tmp_dir}"
-  return ${return_value}
-
-}
-
-# expand_version $column_num
-# add "0:" to version in $colum_num-th column if no ":" is there (epoch)
-# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash)
-
-expand_version() {
-  local column_num
-  column_num="$1"
-
-  sed '
-    /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/
-    /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/
-  '
-}
-
-# shrink_version $column_num
-# remove "0:" from version in $colum_num-th column (epoch)
-# remove "+0" from version in $colum_num-th column (git count/hash)
-
-shrink_version() {
-  local column_num
-  column_num="$1"
-
-  sed '
-    s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/
-    s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/
-  '
-}
-
-# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ]
-
-# Return (to stdout) the biggest subset A of the packages in $omega whose
-# runtime dependencies in $omega \cup $keep are also in A
-
-# $all_builds and $all_depends either point to an empty file - then they will get
-# filled with cached data for subsequent calls - or to the same files of a previous
-# call
-
-# If non-empty, $force contains packages which are assumed to match the above
-# condition without checking.
-
-# The arguments are names of files with one $package.$revision.$mod_revision.$repository
-# per line.
-
-find_biggest_subset_of_packages() {
-
-  ( # the new shell is intentional
-    omega="$1"
-    keep="$2"
-    all_builds="$3"
-    all_depends="$4"
-    if [ $# -eq 4 ]; then
-      force='/dev/null'
-    elif [ $# -eq 5 ]; then
-      force="$5"
-    else
-      >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#"
-      return 2
-    fi
-
-    if [ ! -s "${all_builds}" ]; then
-      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \
-        -exec sed '
-          s|^|{} |
-          s|^\S\+/||
-          s|\.builds | |
-        ' {} \; | \
-        sort -k2,2 > \
-        "${all_builds}"
-    fi
-
-    if [ ! -s "${all_depends}" ]; then
-      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \
-        -exec sed '
-          s|^|{} |
-          s|^\S\+/||
-          s|\.run-depends | |
-        ' {} \; | \
-        grep -v ' base$' | \
-        sort -k2,2 > \
-        "${all_depends}"
-    fi
-
-    sort -u "${omega}" | \
-      sponge "${omega}"
-
-    temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
-    {
-      sort -u "${keep}"
-      cat "${force}" "${force}"
-    } | \
-      sort | \
-      uniq -u > \
-      "${temp_dir}/keep.new"
-    touch "${temp_dir}/keep"
-
-    while [ -s "${temp_dir}/keep.new" ]; do
-      cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \
-        sort -u | \
-        sponge "${temp_dir}/keep"
-
-      {
-        # we append all packages which are run-dependencies of keep-packages
-        # to the keep-list
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.run-depends|
-        ' "${temp_dir}/keep" | \
-          xargs -r grep -HF '' | \
-          sed '
-            s|^.*/||
-            s|\.run-depends:| |
-          ' | \
-          sort -u | \
-          sort -k2,2 | \
-          uniq -f1 | \
-          join -1 2 -2 2 -o 2.1 - "${all_builds}"
-
-        # we append all packages with run-dependencies on the keep-list
-        # to the keep-list
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.builds|
-        ' "${temp_dir}/keep" | \
-          xargs -r grep -HF '' | \
-          sed '
-            s|^.*/||
-            s|\.builds:| |
-          ' | \
-          sort -u | \
-          sort -k2,2 | \
-          uniq -f1 | \
-          join -1 2 -2 2 -o 2.1 - "${all_depends}"
-      } | \
-        sort -u | \
-        join -1 1 -2 1 -o 2.1 - "${omega}" | \
-        sort -u > \
-        "${temp_dir}/keep.new"
-
-      # "new" is only what has not been there before and what is not forced
-      cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \
-        sort | \
-        uniq -u | \
-        sponge "${temp_dir}/keep.new"
-    done
-
-    cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \
-      sort | \
-      uniq -u
-
-  )
-
-}
-
-# sort_square_bracket_content $file
-# sort the content of [] in $file, print to stdout
-
-sort_square_bracket_content() {
-  local file
-  local line
-  local token
-  local token_list
-  local rest
-  file="$1"
-
-  while read -r line; do
-    printf '%s ' "${line}" | \
-      tr ' ' '\n' | \
-      while read -r token; do
-        if echo "${token}" | \
-          grep -qF '['; then
-          printf '%s[' "${token%[*}"
-          token="${token##*[}"
-          token_list="${token%,}"
-          while ! echo "${token_list}" | \
-            grep -qF ']'; do
-            read -r token
-            token_list=$(
-              printf '%s\n' \
-                "${token_list}" \
-                "${token%,}"
-            )
-          done
-          rest="]${token_list#*]}"
-          token_list="${token_list%%]*}"
-          token=$(
-            printf '%s' "${token_list}" | \
-              sort | \
-              sed 's|$|,|'
-            printf '%s' "${rest}"
-          )
-        fi
-        printf '%s\n' "${token}"
-      done | \
-      tr '\n' ' ' | \
-      sed '
-        s|, ]|]|g
-        s| $||
-      '
-    printf '\n'
-  done < \
-    "${file}"
-}
-
-# smoothen_namcap_log $file
-# remove unneccesary differences from namcap-logs:
-#   - remove architecture specific information
-#   - sort lines
-#   - sort content of square brackets
-
-smoothen_namcap_log() {
-  local file
-  file="$1"
-  # shellcheck disable=SC2016
-  sort_square_bracket_content "${file}" | \
-    sed '
-      # normalize architecture specific information
-      s|i[34567]86|$ARCH|g
-      s|x86\([-_]64\)\?|$ARCH|g
-      # remove haskell hashes
-      s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g
-    ' | \
-    sort | \
-    sponge "${file}"
-}
-
-# print_list_of_archaic_packages $source1 $source2 ...
-# print a list of packages which have not been touched for a while,
-# but which are still in the pipeline, e.g. in $source1, $source2 or ...
-
-print_list_of_archaic_packages() {
-  for source in "$@"; do
-    case "${source}" in
-      'testing')
-        # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \
-          -exec head -n1 {} \; | \
-          "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin
-        # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated
-        # and no longer block other packages from being moved
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \
-          sed '
-            s|\.testing$||
-          '
-      ;;
-      'build-list')
-        while read -r pkg rev mod_rev repo; do
-          git_repo=$(
-            find_repository_with_commit "${rev}"
-          )
-          eval repo_path='"${repo_paths__'"${git_repo}"'}"'
-          commit_date=$(
-            git -C "${repo_path}" show -s --format=%ct "${rev}"
-          )
-          mod_commit_date=$(
-            git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}"
-          )
-          if [ "${mod_commit_date}" -gt "${commit_date}" ]; then
-            commit_date="${mod_commit_date}"
-          fi
-          # packages remaining longer than $max_package_age_build_list days on the build list
-          if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then
-            printf '%s %s %s %s\n' \
-              "${pkg}" \
-              "${rev}" \
-              "${mod_rev}" \
-              "${repo}"
-          fi
-        done < \
-          "${work_dir}/build-list"
-      ;;
-      'staging')
-        # packages remaining longer than $max_package_age_staging days in staging
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \
-          sed '
-            s|\.done$||
-          '
-      ;;
-      *)
-        >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}"
-      ;;
-    esac
-  done | \
-    sort -u
-}
-
-# modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
-# print the given modification revision possibly with a html link to github
-
-modification_revision_link() {
-  local mod_rev="$1"
-  local repo="$2"
-  local pkg="$3"
-
-  if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then
-    printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \
-      "${mod_rev}" \
-      "${repo}" \
-      "${pkg}" \
-      "${mod_rev}"
-  else
-    printf '%s\n' \
-      "${mod_rev}"
-  fi
-}
-
-# trigger_mirror_refreshs
-# trigger a refresh of capable tier 1 mirrors (as backup for master mirror)
-
-trigger_mirror_refreshs() {
-  local tmp_file
-
-  tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir)
-  date '+%s' > \
-    "${tmp_file}"
-  ${master_mirror_rsync_command} \
-    "${tmp_file}" \
-    "${master_mirror_rsync_directory}/lastupdate"
-  rm "${tmp_file}"
-  for trigger_url in ${mirror_refresh_trigger_urls}; do
-    screen -S trigger-mirror-update -d -m curl -L "${trigger_url}"
-  done
-}
-
-# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name
-extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() {
-  pkgname="$1"
-  pkgname="${pkgname%.pkg.tar.xz}"
-  arch="${pkgname##*-}"
-  pkgname="${pkgname%-*}"
-  sub_pkgrel="${pkgname##*-}"
-  pkgname="${pkgname%-*}"
-  pkgrel="${sub_pkgrel%.*}"
-  if [ "${pkgrel}" = "${sub_pkgrel}" ]; then
-    sub_pkgrel='0'
-  else
-    sub_pkgrel="${sub_pkgrel##*.}"
-  fi
-  epoch="${pkgname##*-}"
-  pkgname="${pkgname%-*}"
-  pkgver="${epoch#*:}"
-  if [ "${pkgver}" = "${epoch}" ]; then
-    epoch='0'
-  else
-    epoch="${epoch%%:*}"
-  fi
-}
diff --git a/bin/mysql-functions b/bin/mysql-functions
deleted file mode 100755
index 185d9a9..0000000
--- a/bin/mysql-functions
+++ /dev/null
@@ -1,1098 +0,0 @@
-#!/bin/sh
-
-# contains functions used to access mysql db
-
-# shellcheck disable=SC2016,SC2039
-
-if [ -z "${base_dir}" ]; then
-  # just to make shellcheck happy
-  . 'conf/default.conf'
-fi
-
-# base64_encode_each encode each line of stdin with base64
-
-base64_encode_each() {
-  local line
-
-  while read -r line; do
-    printf '%s' \
-      "${line}" | \
-      base64 -w0
-    printf '\n'
-  done
-}
-
-# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
-
-# shellcheck disable=SC2086
-mysql_add_package_source() {
-  local names='pkgbase git_revision mod_git_revision upstream_package_repository'
-  local values
-  local uses_upstream
-  local uses_modification
-  local repo
-
-  if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then
-    uses_modification=1
-  else
-    uses_modification=0
-  fi
-  uses_upstream=0
-  for repo in ${repo_names}; do
-    if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
-      tar -t 2>/dev/null | \
-      sed 's,-x86_64/,-any/,' | \
-      grep -qFx "$1/repos/$4-any/PKGBUILD"; then
-      uses_upstream=1
-    fi
-  done
-
-  for _ in ${names}; do
-    values="${values}$(
-      printf '%s' "$1" | \
-        base64 -w0
-    ) "
-    shift
-  done
-  values="${values% }"
-
-  {
-    printf 'INSERT IGNORE INTO package_sources'
-    printf ' ('
-    printf '`%s`, ' ${names}
-    printf '`uses_upstream`,`uses_modification`'
-    printf ') SELECT'
-    printf ' from_base64("%s"), ' ${values% *}
-    printf ' `upstream_repositories`.`id`,%s,%s' \
-      ${uses_upstream} ${uses_modification}
-    printf ' FROM `upstream_repositories`'
-    printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \
-      "${values##* }"
-  } | \
-    ${mysql_command}
-}
-
-# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository
-
-# shellcheck disable=SC2031,SC2086,SC2154
-mysql_add_binary_package() {
-  local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository'
-  local name
-  for name in ${names}; do
-    eval 'local '"${name}"
-    eval "${name}"'=$(
-      printf "%s" "$1" |
-        base64 -w0
-    )'
-    shift
-  done
-
-  {
-    printf 'INSERT IGNORE INTO binary_packages'
-    printf ' ('
-    printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted'
-    printf ') SELECT'
-    printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}"
-    printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures'
-    printf ' 0, 0, 0'
-    printf ' FROM'
-    printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures'
-    printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
-    printf ' WHERE'
-    printf ' `%s`.`name` = from_base64("%s") AND' \
-      'repositories' "${repository}" \
-      'architectures' "${architecture}"
-    printf ' `package_sources`.`%s` = from_base64("%s") AND' \
-      'pkgbase' "${pkgbase}" \
-      'git_revision' "${git_revision}" \
-      'mod_git_revision' "${mod_git_revision}"
-    printf ' `upstream_repositories`.`name` = from_base64("%s")' \
-      "${upstream_package_repository}"
-  } | \
-    sed '
-      s|, )|)|g
-      s|, FROM| FROM|g
-    ' | \
-    ${mysql_command}
-}
-
-# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel
-
-# shellcheck disable=SC2031,SC2086,SC2154
-mysql_show_binary_package() {
-  local names='pkgname pkgver pkgrel sub_pkgrel'
-  local name
-  for name in ${names}; do
-    eval 'local '"${name}"
-    eval "${name}"'=$(
-      printf "%s" "$1" |
-        base64 -w0
-    )'
-    shift
-  done
-
-  {
-    printf 'SELECT'
-    printf ' `%s`.`%s`,' \
-      'repositories' 'name' \
-      'binary_packages' 'pkgname' \
-      'package_sources' 'pkgver' \
-      'package_sources' 'pkgrel' \
-      'binary_packages' 'sub_pkgrel' \
-      'architectures' 'name' \
-      'package_sources' 'pkgbase' \
-      'package_sources' 'git_revision' \
-      'package_sources' 'mod_git_revision' \
-      'upstream_repositories' 'name'
-    printf ' FROM `binary_packages`'
-    mysql_join_binary_packages_architectures
-    mysql_join_binary_packages_repositories
-    mysql_join_binary_packages_build_assignments
-    mysql_join_build_assignments_package_sources
-    mysql_join_package_sources_upstream_repositories
-    printf ' WHERE'
-    printf ' `%s`.`%s` = from_base64("%s") AND' \
-      'binary_packages' 'pkgname' "${pkgname}" \
-      'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \
-      'package_sources' 'pkgver' "${pkgver}" \
-      'package_sources' 'pkgrel' "${pkgrel}"
-    printf ';'
-  } | \
-    sed '
-      s|, FROM| FROM|g
-      s|AND;|;|g
-    ' | \
-    ${mysql_command} --html
-}
-
-# mysql_add_install_target $install_target
-
-# shellcheck disable=2086
-mysql_add_install_target() {
-  local install_target
-  install_target=$(
-    printf "%s" "$1" | \
-      base64 -w0
-  )
-
-  {
-    printf 'INSERT IGNORE INTO `install_targets` (`name`)'
-    printf ' VALUES (from_base64("%s"))' \
-      "${install_target}"
-  } | \
-    ${mysql_command}
-}
-
-# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository
-# or
-# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository
-# if sub_pkgrel should be determined automatically
-# and
-# mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository
-# or
-# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository
-# if $sub_pkgrel should be forced
-
-# generate the meta data of a package (dependencies, built packages, ...) in the database
-
-mysql_generate_package_metadata() {
-
-  ( # new shell is intentional
-    case "$1" in
-      ''|*[!0-9]*)
-        unset forced_sub_pkgrel
-      ;;
-      *)
-        forced_sub_pkgrel=$(
-          printf '%s' "$1" | \
-            base64 -w0
-        )
-        shift
-      ;;
-    esac
-    current_repository="$1"
-    package="$2"
-
-    if [ $# -eq 2 ]; then
-      # second form
-      repository="${package##*.}"
-      package="${package%.*}"
-      mod_git_revision="${package##*.}"
-      package="${package%.*}"
-      git_revision="${package##*.}"
-      package="${package%.*}"
-    else
-      git_revision="$3"
-      mod_git_revision="$4"
-      repository="$5"
-    fi
-
-    temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
-    printf '.' >&2
-    if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then
-      printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"
-      exit 2
-    fi
-    # remove empty lines and unsupported architectures
-    sed -i '
-      /^[^=]*=\s*$/d
-      /^\s*arch = /{
-        / \(i686\|any\)$/!d
-      }
-    ' "${temp_dir}/SRCINFO"
-
-    if [ ! -s "${temp_dir}/SRCINFO" ]; then
-      >&2 printf '"make_source_info" had empty output - eh, what?\n'
-      exit 2
-    fi
-    printf '\n\n' >> "${temp_dir}/SRCINFO"
-
-    printf '.' >&2
-    pkgbase=$(
-      grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \
-        cut -d' ' -f3
-    )
-    if [ -z "${pkgbase}" ]; then
-      >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n'
-      exit 2
-    fi
-
-    # add the package source
-    mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
-    printf '.' >&2
-
-    # now we encode everything in base64
-    current_repository=$(
-      printf '%s' "${current_repository}" | \
-        base64 -w0
-    )
-    pkgbase=$(
-      printf '%s' "${pkgbase}" | \
-        base64 -w0
-    )
-    git_revision=$(
-      printf '%s' "${git_revision}" | \
-        base64 -w0
-    )
-    mod_git_revision=$(
-      printf '%s' "${mod_git_revision}" | \
-        base64 -w0
-    )
-    repository=$(
-      printf '%s' "${repository}" | \
-        base64 -w0
-    )
-
-    # add the build assignment(s)
-    {
-      archs=$(
-        sed -n '
-          s/^\tarch = //
-          T
-          p
-        ' "${temp_dir}/SRCINFO" | \
-          grep -vxF 'any' | \
-          sort -u
-      )
-      if [ -z "${archs}" ]; then
-        echo 'any'
-      else
-        printf '%s\n' "${archs}"
-      fi
-    } | \
-      while read -r arch; do
-        printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
-        printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0'
-        printf ' FROM `architectures` JOIN `package_sources`'
-        printf ' WHERE `architectures`.`name` = from_base64("%s")' \
-          "$(
-            printf '%s' "${arch}" | \
-              base64 -w0
-          )"
-        printf ' AND `package_sources`.`%s` = from_base64("%s")' \
-          'pkgbase' "${pkgbase}" \
-          'git_revision' "${git_revision}" \
-          'mod_git_revision' "${mod_git_revision}"
-        printf ';\n'
-      done > \
-      "${temp_dir}/add-build-assignments-command"
-
-    # TODO: correctly link between binary_packages and build_assignments using any_arch
-
-    # shellcheck disable=SC2034
-    # select any specific arch (which will be building the 'any' part of a split package)
-    any_arch=$(
-      {
-        sed -n '
-          s/^\tarch = //
-          T
-          p
-        ' "${temp_dir}/SRCINFO" | \
-          sort -r | \
-          grep -vxFm 1 'any' || \
-          echo 'any'
-      } | \
-        base64_encode_each
-    )
-
-    grep '^pkgname = ' "${temp_dir}/SRCINFO" | \
-      cut -d' ' -f3 | \
-      while read -r pkgname; do
-        pkgname64=$(
-          printf '%s' "${pkgname}" | \
-            base64 -w0
-        )
-        sed -n '
-          /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{
-            :a
-              N
-              /\n$/{
-                p
-                T
-              }
-              ba
-          }
-        ' "${temp_dir}/SRCINFO" | \
-          sed '
-            /^\S/d
-            s/^\s*//
-          ' > \
-          "${temp_dir}/BINARYINFO.${pkgname64}"
-
-        grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \
-          cut -d' ' -f3 | \
-          while read -r arch; do
-            arch64=$(
-              printf '%s' "${arch}" | \
-                base64 -w0
-            )
-            sed '
-              s/^\(\S\+\)_'"${arch}"' = /\1 = /
-            ' "${temp_dir}/BINARYINFO.${pkgname64}" > \
-              "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}"
-          done
-      done
-    find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \
-      while read -r _ pkgname arch; do
-        pkgver=$(
-          grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            base64_encode_each
-        )
-        pkgrel=$(
-          grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            base64_encode_each
-        )
-        epoch=$(
-          {
-            grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \
-              echo 'epoch = 0'
-          } | \
-            cut -d' ' -f3 | \
-            base64_encode_each
-        )
-        provides=$(
-          grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
-        )
-        makedepends=$(
-          grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
-        )
-        checkdepends=$(
-          grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
-        )
-        rundepends=$(
-          grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
-        )
-        if [ -n "${forced_sub_pkgrel}" ]; then
-          sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")'
-        else
-          sub_pkgrel=$(
-            printf '(SELECT COALESCE('
-            # do not add binary packages which are currently on the
-            # build-list or in $current_repository (beware of split
-            # packages!)
-            printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
-            mysql_join_binary_packages_architectures
-            mysql_join_binary_packages_repositories
-            printf ' WHERE'
-            printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-              'epoch' "${epoch}" \
-              'pkgver' "${pkgver}" \
-              'pkgrel' "${pkgrel}" \
-              'pkgname' "${pkgname}"
-            printf ' `architectures`.`name`=from_base64("%s")' \
-              "${arch}"
-            printf ' AND `repositories`.`name` IN ("build-list",from_base64("%s"))),' \
-              "${current_repository}"
-            # max(sub_pkgrel)+1
-            printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
-            mysql_join_binary_packages_architectures
-            printf ' WHERE'
-            printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-              'epoch' "${epoch}" \
-              'pkgver' "${pkgver}" \
-              'pkgrel' "${pkgrel}" \
-              'pkgname' "${pkgname}"
-            if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then
-              # 'any' gets higher sub_pkgrel than any architecture
-              printf ' 1'
-            else
-              # not-'any' gets higher sub_pkgrel than same or 'any' architecture
-              printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \
-                "${arch}"
-            fi
-            printf ')'
-            printf ',0))'
-          )
-        fi
-        {
-          printf 'INSERT IGNORE INTO `binary_packages` ('
-          printf '`%s`,' \
-            'build_assignment' \
-            'repository' \
-            'architecture' \
-            'epoch' \
-            'pkgver' \
-            'pkgrel' \
-            'pkgname' \
-            'sub_pkgrel' \
-            'has_issues' \
-            'is_tested' \
-            'is_to_be_deleted'
-          printf ') SELECT '
-          printf '`%s`.`id`,' \
-            'build_assignments' \
-            'repositories' \
-            'architectures'
-          printf 'from_base64("%s"),' \
-            "${epoch}" \
-            "${pkgver}" \
-            "${pkgrel}" \
-            "${pkgname}"
-          printf '%s,0,0,0 FROM' \
-            "${sub_pkgrel}"
-          printf ' `%s` JOIN' \
-            'repositories' \
-            'architectures' \
-            'build_assignments'
-          mysql_join_build_assignments_package_sources
-          mysql_join_package_sources_upstream_repositories
-          printf ' WHERE'
-          printf ' `%s`.`%s` = from_base64("%s") AND' \
-            'repositories' 'name' "${current_repository}" \
-            'architectures' 'name' "${arch}" \
-            'package_sources' 'pkgbase' "${pkgbase}" \
-            'package_sources' 'git_revision' "${git_revision}" \
-            'package_sources' 'mod_git_revision' "${mod_git_revision}" \
-            'upstream_repositories' 'name' "${repository}"
-          printf ';\n'
-        } | \
-          sed '
-            s|,)|)|g
-            s| JOIN JOIN | JOIN |
-            s| AND;$|;|
-          ' >> \
-          "${temp_dir}/add-binary-packages-command"
-        {
-          printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \
-            'provides' \
-            'makedepends' \
-            'checkdepends' \
-            'rundepends'
-
-          printf 'INSERT INTO `provides` VALUES\n'
-          echo "${provides}" | \
-            sort -u | \
-            grep -vxF '' | \
-            sed '
-              s|^| (from_base64("|
-              s|$|")),|
-            '
-          printf ' (from_base64("%s"));\n' \
-            "${pkgname}"
-
-          printf 'INSERT INTO `rundepends` VALUES\n'
-          echo "${rundepends}" | \
-            sort -u | \
-            grep -vxF '' | \
-            sed '
-              s|^| (from_base64("|
-              s|$|")),|
-            '
-          printf ' ("base");\n'
-
-          echo "${checkdepends}" | \
-            sort -u | \
-            grep -vxF '' | \
-            sed '
-              s|^| (from_base64("|
-              s|$|")),|
-              1 s/^/INSERT INTO `checkdepends` VALUES \n/
-              $ s/,$/;/
-            '
-
-          printf 'INSERT INTO `makedepends` VALUES\n'
-          echo "${makedepends}" | \
-            sort -u | \
-            grep -vxF '' | \
-            sed '
-              s|^| (from_base64("|
-              s|$|")),|
-            '
-          printf ' ("base-devel");\n'
-
-          printf 'INSERT IGNORE INTO `install_targets` (`name`)'
-          printf ' SELECT (`name`) FROM `%s` UNION' \
-            'provides' \
-            'makedepends' \
-            'checkdepends' \
-            'rundepends' | \
-            sed 's| UNION$|;\n|'
-
-          for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
-            case "${link}" in
-              'provides')
-                printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT'
-                printf ' `binary_packages`.`id`,`install_targets`.`id` FROM'
-              ;;
-              'makedepends'|'checkdepends'|'rundepends')
-                printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT'
-                printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM'
-                printf ' `dependency_types` JOIN'
-              ;;
-            esac
-            printf ' `binary_packages`'
-            mysql_join_binary_packages_architectures
-            printf ' JOIN `install_targets`'
-            printf ' JOIN `%s`' "${link}"
-            printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
-            printf ' WHERE'
-            if [ "${link}" = 'makedepends' ] || \
-              [ "${link}" = 'checkdepends' ] || \
-              [ "${link}" = 'rundepends' ]; then
-              printf ' `dependency_types`.`name` = "%s" AND' \
-                "${link%depends}"
-            fi
-            printf ' `binary_packages`.`%s` = from_base64("%s") AND' \
-              'epoch' "${epoch}" \
-              'pkgver' "${pkgver}" \
-              'pkgrel' "${pkgrel}" \
-              'pkgname' "${pkgname}"
-            # we do not want to match the sub_pkgrel:
-            # a) it is tedious to do so (because it may be calculated
-            #    dynamically)
-            # b) it is not necessary to do so: if only the sub_pkgrel
-            #    changed, the dependencies and provided install_targets
-            #    should not have changed
-            printf ' `architectures`.`name` = from_base64("%s");\n' \
-              "${arch}"
-            # the repository is of no relevance: it hardly matters for
-            # the dependencies
-          done
-
-          printf 'DROP TABLE `%s`;\n' \
-            'provides' \
-            'makedepends' \
-            'checkdepends' \
-            'rundepends'
-        } >> \
-          "${temp_dir}/add-install-targets-command"
-      done
-    printf '.' >&2
-
-    {
-      if [ -s "${temp_dir}/add-build-assignments-command" ]; then
-        cat "${temp_dir}/add-build-assignments-command"
-      fi
-      if [ -s "${temp_dir}/add-binary-packages-command" ]; then
-        cat "${temp_dir}/add-binary-packages-command"
-      fi
-      if [ -s "${temp_dir}/add-install-targets-command" ]; then
-        cat "${temp_dir}/add-install-targets-command"
-      fi
-    } | \
-      ${mysql_command}
-    printf '.' >&2
-
-  )
-}
-
-# mysql_sanity_check
-# do a sanity check on the mysql database
-
-mysql_sanity_check() {
-  {
-    printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)'
-    printf ' FROM `binary_packages`'
-    mysql_join_binary_packages_build_assignments
-    mysql_join_binary_packages_architectures '' 'bp_arch'
-    mysql_join_build_assignments_architectures '' 'ba_arch'
-    printf ' WHERE `bp_arch`.`name`!="any"'
-    printf ' AND `ba_arch`.`name`="any";\n'
-    printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)'
-    printf ' FROM `binary_packages` AS `a`'
-    printf ' JOIN `binary_packages` AS `b`'
-    printf ' ON `a`.`pkgname`=`b`.`pkgname`'
-    printf ' AND `a`.`repository`=`b`.`repository`'
-    printf ' AND `a`.`id`!=`b`.`id`'
-    mysql_join_binary_packages_repositories 'a'
-    printf ' WHERE `repositories`.`name`="build-list";\n'
-    printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)'
-    printf ' FROM `binary_packages` AS `a`'
-    printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`'
-    mysql_join_binary_packages_repositories 'a' 'arep'
-    mysql_join_binary_packages_repositories 'b' 'brep'
-    printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`'
-    printf ' AND `%srep`.`name`="build-list"' \
-      'a' 'b'
-    printf ';\n'
-  } | \
-    ${mysql_command} --raw --batch | \
-    sed '
-      /^CONCAT("/d
-      s,^,<font color="#FF0000">,
-      s,$,</font>,
-    '
-  ( # new shell is intentional
-    temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
-    for dir in $(ls_master_mirror 'i686'); do
-      ls_master_mirror "i686/${dir}" | \
-        sed -n '
-          s/\.pkg\.tar\.xz$//
-          T
-          s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/
-          s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/
-          s|^|'"${dir}"'/|
-          p
-        '
-    done | \
-      sort > \
-      "${temp_dir}/master-mirror-listing"
-
-    {
-      printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`'
-      printf ' FROM `binary_packages`'
-      mysql_join_binary_packages_architectures
-      mysql_join_binary_packages_repositories
-      printf ' WHERE `repositories`.`is_on_master_mirror`'
-    } | \
-      ${mysql_command} --batch | \
-      sed '
-        1d
-        s,\t,/,
-        s,\t,-,
-        s,\t,:,
-        s,\t,-,
-        s,\t,.,
-        s,\t,-,
-      ' | \
-      sort > \
-      "${temp_dir}/mysql-packages"
-
-    diff -u \
-      "${temp_dir}/master-mirror-listing" \
-      "${temp_dir}/mysql-packages"
-  )
-}
-
-mysql_find_build_assignment_loops() {
-  new_loops=$(
-    {
-      printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`'
-      printf ' FROM `dependencies`'
-      mysql_join_dependencies_install_target_providers
-      mysql_join_install_target_providers_binary_packages '' 'packages_dependency'
-      mysql_join_dependencies_binary_packages '' 'packages_dependent'
-      mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency'
-      mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent'
-      printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
-    } | \
-      ${mysql_command} --raw --batch | \
-      sed '
-        1d
-        y/\t/ /
-      ' | \
-      tsort 2>&1 >/dev/null | \
-      sed 's/^tsort:\s*//' | \
-      {
-        loop=0
-        while read -r id; do
-          if [ "x${id}" = 'x-: input contains a loop:' ]; then
-            loop=$((loop+1))
-            continue
-          fi
-          if ! printf '%s' "${id}" | tr '\n' ' ' | grep -q '^[0-9]\+$'; then
-            >&2 printf 'ERROR: non-numeric id "%s"\n' "${id}"
-            continue
-          fi
-          printf '(%s,%s),' "${loop}" "${id}"
-        done | \
-          sed 's/,$//'
-      }
-  )
-  {
-    printf 'DELETE FROM `build_dependency_loops`;\n'
-    if [ -n "${new_loops}" ]; then
-      printf 'INSERT INTO `build_dependency_loops` (`loop`,`build_assignment`) VALUES %s;\n' \
-        "${new_loops}"
-    fi
-  } | \
-    ${mysql_command}
-}
-
-# mysql_cleanup [dry]
-# clean up left overs from mysql database
-mysql_cleanup() {
-  local operator
-  if [ "$#" = '0' ]; then
-    operator='DELETE'
-  elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then
-    operator='SELECT COUNT(*)'
-  else
-    >&2 echo 'Unknown parameter'
-    >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".'
-    exit 2
-  fi
-  {
-    # remove to-be-decided binary_packages
-    printf '%s ' \
-      "${operator}"
-    if [ "${operator}" = 'DELETE' ]; then
-      printf '`binary_packages` '
-    fi
-    printf 'FROM `binary_packages`'
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="to-be-decided";\n'
-    # remove dependencies w/o binary_package or install_target
-    printf '%s FROM `dependencies` ' \
-      "${operator}"
-    printf 'WHERE NOT EXISTS '
-    printf '('
-      printf 'SELECT * FROM `binary_packages` '
-      printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`'
-    printf ') OR NOT EXISTS '
-    printf '('
-      printf 'SELECT * FROM `install_targets` '
-      printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`'
-    printf ');\n'
-    # remove install_target_providers w/o binary_package or install_target
-    printf '%s FROM `install_target_providers` ' \
-      "${operator}"
-    printf 'WHERE NOT EXISTS '
-    printf '('
-      printf 'SELECT * FROM `binary_packages` '
-      printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`'
-    printf ') OR NOT EXISTS '
-    printf '('
-      printf 'SELECT * FROM `install_targets` '
-      printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`'
-    printf ');\n'
-    # remove build_assignments w/o binary_package
-    printf '%s FROM `build_assignments` ' \
-      "${operator}"
-    printf 'WHERE NOT EXISTS '
-    printf '('
-      printf 'SELECT * FROM `binary_packages` '
-      printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`'
-    printf ');\n'
-    # remove failed_builds with unbroken build_assignments
-    printf '%s ' \
-      "${operator}"
-    if [ "${operator}" = 'DELETE' ]; then
-      printf '`failed_builds` '
-    fi
-    printf 'FROM `failed_builds` '
-    mysql_join_failed_builds_build_assignments
-    printf 'WHERE NOT `build_assignments`.`is_broken`'
-    printf ';\n'
-    # remove failed_builds w/o build_assignment
-    printf '%s FROM `failed_builds` ' \
-      "${operator}"
-    printf 'WHERE NOT EXISTS '
-    printf '('
-      printf 'SELECT * FROM `build_assignments` '
-      printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`'
-    printf ');\n'
-    # remove package_sources w/o build_assignment
-    printf '%s FROM `package_sources` ' \
-      "${operator}"
-    printf 'WHERE NOT EXISTS '
-    printf '('
-      printf 'SELECT * FROM `build_assignments` '
-      printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`'
-    printf ');\n'
-    printf 'UPDATE `build_slaves`'
-    mysql_join_build_slaves_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' SET `currently_building`=NULL'
-    printf ' WHERE `repositories`.`name`!="build-list";\n'
-  } | \
-    ${mysql_command}
-}
-
-# mysql_query_has_pending_dependencies `build_assignment`.`id`
-# print a mysql query giving wether dependencies are pending
-mysql_query_has_pending_dependencies() {
-  printf 'EXISTS ('
-    printf 'SELECT * FROM `binary_packages` as `to_dos`'
-    mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos'
-    mysql_join_binary_packages_dependencies 'to_dos'
-    mysql_join_dependencies_install_target_providers
-    mysql_join_install_target_providers_binary_packages '' 'bin_deps'
-    mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos'
-    printf ' WHERE'
-    printf ' `%s`.`name`="build-list" AND' \
-      'dep_repos' 'to_do_repos'
-    printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND'
-    printf ' `to_dos`.`build_assignment`=%s' \
-      "$1"
-  printf ')'
-}
-
-# mysql_query_is_part_of_loop `build_assignment`.`id`
-# print a mysql query giving wether the package is part of a loop
-mysql_query_is_part_of_loop() {
-  printf 'EXISTS ('
-    printf 'SELECT * FROM `build_dependency_loops`'
-    printf ' WHERE `build_dependency_loops`.`build_assignment`=%s' \
-      "$1"
-  printf ')'
-}
-
-# mysql_query_select_pkgbase_and_revision
-# print the part of a mysql query giving:
-# pkgbase git_revision mod_git_revision upstream_package_repository
-mysql_query_select_pkgbase_and_revision() {
-  printf '`package_sources`.`%s`,' \
-    'pkgbase' \
-    'git_revision' \
-    'mod_git_revision'
-  printf '`upstream_repositories`.`name`'
-  printf ' FROM `build_assignments`'
-  mysql_join_build_assignments_package_sources
-  mysql_join_package_sources_upstream_repositories
-}
-
-# mysql_repair_binary_packages_without_build_assignment
-# try to generate valid build assignments to binary packages without
-# a valid one yet
-mysql_repair_binary_packages_without_build_assignment() {
-  {
-    printf 'SELECT '
-    printf '`binary_packages`.`id`'
-    printf ',replace(to_base64(%s),"\\n","")' \
-      '`binary_packages`.`pkgname`' \
-      '`architectures`.`name`'
-    printf ' FROM `binary_packages`'
-    mysql_join_binary_packages_architectures
-    printf ' WHERE `binary_packages`.`build_assignment`<0'
-  } | \
-    ${mysql_command} --raw --batch | \
-    sed '1d' | \
-    while read -r id pkgname arch; do
-      pkgname=$(
-        printf '%s' "${pkgname}" | \
-          base64 -d
-      )
-      pkgbase=$(
-        curl -Ss "$(
-          printf 'https://www.archlinux.org/packages/search/json/?name=%s' \
-            "${pkgname}"
-        )" | \
-          sed '
-            s/^.*"results":\s*\[//
-            s/}\s*,\s*{/\n/g
-          ' | \
-          grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \
-          tr ',' '\n' | \
-          grep '"pkgbase":' | \
-          cut -d'"' -f4 | \
-          sort -u | \
-          head -n1
-      )
-      if [ -z "${pkgbase}" ] && \
-        {
-          printf 'SELECT count(*) FROM `package_sources`'
-          printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
-            "$(printf '%s' "${pkgname}" | base64 -w0)"
-        } | \
-          ${mysql_command} --raw --batch | \
-          sed '1d' | \
-          grep -qvxF '0'; then
-        pkgbase="${pkgname}"
-      fi
-      if [ -z "${pkgbase}" ]; then
-        >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}"
-        continue
-      fi
-      pkgbase=$(
-        printf '%s' "${pkgbase}" | \
-          base64 -w0
-      )
-      printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
-      printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0'
-      printf ' FROM `package_sources`'
-      printf ' JOIN `architectures`'
-      printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}"
-      printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}"
-      printf ' LIMIT 1;\n'
-      printf 'UPDATE `binary_packages`'
-      printf ' JOIN `build_assignments`'
-      mysql_join_binary_packages_build_assignments
-      printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`'
-      printf ' WHERE `binary_packages`.`id`=%s' "${id}"
-      printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}"
-    done | \
-    ${mysql_command}
-}
-
-# mysql_remove_duplicate_binary_packages
-# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel,
-# having difference of 1 in sub_pkgrel
-
-mysql_remove_duplicate_build_order() {
-  {
-    printf 'CREATE TEMPORARY TABLE `ren`'
-    printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n'
-    printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)'
-    printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`'
-    printf ' FROM `binary_packages` as `old`'
-    printf ' JOIN `binary_packages` as `new` ON'
-    printf ' `old`.`%s`=`new`.`%s` AND' \
-      'pkgname' 'pkgname' \
-      'epoch' 'epoch' \
-      'pkgver' 'pkgver' \
-      'pkgrel' 'pkgrel'
-    printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`'
-    mysql_join_binary_packages_repositories 'old' 'orep'
-    mysql_join_binary_packages_repositories 'new' 'nrep'
-    printf ' WHERE `orep`.`name`!="to-be-decided"'
-    printf ' AND `nrep`.`name`="to-be-decided";\n'
-    printf 'UPDATE IGNORE `dependencies`'
-    printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`'
-    printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n'
-    printf 'UPDATE IGNORE `install_target_providers`'
-    printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`'
-    printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n'
-    printf 'DELETE FROM `binary_packages`'
-    printf ' WHERE EXISTS ('
-      printf 'SELECT * FROM `ren`'
-      printf ' WHERE `ren`.`old`=`binary_packages`.`id`'
-    printf ');\n'
-    printf 'UPDATE IGNORE `binary_packages`'
-    printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`'
-    printf ' SET `binary_packages`.`repository`=`ren`.`repo`,'
-    printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n'
-  } | \
-    ${mysql_command}
-}
-
-# mysql_package_name_query
-# print a mysql query of the full name of a package file
-
-mysql_package_name_query() {
-  printf 'CONCAT('
-  printf '`binary_packages`.`pkgname`,"-",'
-  printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
-  printf '`binary_packages`.`pkgver`,"-",'
-  printf '`binary_packages`.`pkgrel`,".",'
-  printf '`binary_packages`.`sub_pkgrel`,"-",'
-  printf '`architectures`.`name`,".pkg.tar.xz"'
-  printf ')'
-}
-
-# mysql_join_*_*
-# print 'JOIN' part of mysql query to connect the respective tables
-# these functions take 2 optional arguments, acting as aliases for
-# the tables
-
-# mysql_join__generic $table_a $column_a $table_b $column_b
-# create mysql_join_${table_a}_${table_b}() function
-
-mysql_join__generic() {
-  eval "$(
-    printf 'mysql_join_%s_%s() {\n' "$1" "$3"
-    printf '  printf '"'"' JOIN `%s`'"'"'\n' "$3"
-    printf '  if [ -n "$2" ]; then\n'
-    printf '    printf '"'"' AS `%%s`'"'"' "$2"\n'
-    printf '  fi\n'
-    printf '  if [ -n "$1" ]; then\n'
-    printf '    printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2"
-    printf '  else\n'
-    printf '    printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2"
-    printf '  fi\n'
-    printf '  if [ -n "$2" ]; then\n'
-    printf '    printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4"
-    printf '  else\n'
-    printf '    printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4"
-    printf '  fi\n'
-    printf '}\n'
-  )"
-}
-
-for link in \
-  'binary_packages:architecture:architectures' \
-  'binary_packages:repository:repositories' \
-  'binary_packages:build_assignment:build_assignments' \
- \
-  'build_assignments:architecture:architectures' \
-  'build_assignments:package_source:package_sources' \
- \
-  'build_dependency_loops:build_assignment:build_assignments' \
-  'build_dependency_loops:build_assignment build_assignment:binary_packages' \
- \
-  'build_slaves:currently_building:build_assignments' \
-  'build_slaves:currently_building build_assignment:binary_packages' \
- \
-  'dependencies:depending_on:install_targets' \
-  'dependencies:dependent:binary_packages' \
-  'dependencies:dependency_type:dependency_types' \
- \
-  'failed_builds:reason:fail_reason' \
-  'failed_builds:build_assignment:build_assignments' \
-  'failed_builds:build_slave:build_slaves' \
- \
-  'install_target_providers:package:binary_packages' \
-  'install_target_providers:install_target:install_targets' \
-  'install_target_providers:install_target depending_on:dependencies' \
- \
-  'package_sources:upstream_package_repository:upstream_repositories' \
- \
-  'repositories:stability:repository_stabilities' \
- \
-  'upstream_repositories:git_repository:git_repositories'; do
-# A join for these cannot be done, because it's not clear on what to join:
-#  'repository_stability_relations:more_stable:repository_stabilities'
-#  'repository_stability_relations:less_stable:repository_stabilities'
-
-  table_b="${link##*:}"
-  table_a="${link%:*}"
-  column_b="${table_a##*:}"
-  table_a="${table_a%:*}"
-  column_a="${column_b% *}"
-  if [ "${column_a}" = "${column_b}" ]; then
-    column_b='id'
-  else
-    column_b="${column_b##* }"
-  fi
-
-  mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}"
-  mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}"
-done
diff --git a/conf/default.conf b/conf/default.conf
index 84c0e4f..18f5d73 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -14,10 +14,10 @@ else
   base_dir=$(printf '%s/..' "$(dirname "$(readlink -f "$0")")")
 fi
 
-# shellcheck source=bin/common-functions
-. "${base_dir}/bin/common-functions"
-# shellcheck source=bin/mysql-functions
-. "${base_dir}/bin/mysql-functions"
+# shellcheck source=lib/common-functions
+. "${base_dir}/lib/common-functions"
+# shellcheck source=lib/mysql-functions
+. "${base_dir}/lib/mysql-functions"
 
 work_dir="${base_dir}/work"
 
diff --git a/lib/common-functions b/lib/common-functions
new file mode 100755
index 0000000..a84271b
--- /dev/null
+++ b/lib/common-functions
@@ -0,0 +1,1378 @@
+#!/bin/sh
+
+# contains functions used by more than one script
+
+# shellcheck disable=SC2039
+
+# TODO: include link depenendencies in run-depends metadata
+
+# TODO: have full information (currently in files) in database
+
+# TODO: remove state files / metadata files
+
+if [ -z "${base_dir}" ]; then
+  # just to make shellcheck happy
+  . 'conf/default.conf'
+fi
+
+# find_pkgbuilds package repository git_repository git_revision mod_git_revision
+# find the PKGBUILD and modification of $package from $repository
+# sets $PKGBUILD and $PKGBUILD_mod
+
+find_pkgbuilds() {
+
+  local package="$1"
+  local repository="$2"
+  local git_repository="$3"
+  local git_revision="$4"
+  local mod_git_revision="$5"
+
+  local repo_path
+  eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
+
+  PKGBUILD=$(
+    git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \
+      tar -t 2> /dev/null | \
+      grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \
+      grep -v -- '-i686/PKGBUILD$' | \
+      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
+      sort | \
+      tail -n1
+  )
+
+  PKGBUILD_mod=$(
+    git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \
+      tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null
+  ) || true
+
+  if [ -z "${PKGBUILD}" ] && \
+    [ -z "${PKGBUILD_mod}" ]; then
+    >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \
+      "${package}" \
+      "${repository}" \
+      "${git_repository}" \
+      "${git_revision}" \
+      "${mod_git_revision}"
+    return 1
+  fi
+
+}
+
+# find_repository_with_commit commit
+# find the repository which has $commit
+
+find_repository_with_commit() {
+
+  local repository
+
+  for repository in ${repo_names}; do
+    # shellcheck disable=SC2016
+    if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then
+      echo "${repository}"
+      return 0
+    fi
+  done
+  >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1"
+  exit 1
+
+}
+
+# find_git_repository_to_package_repository repository
+# find the git repository which tracks the package repository $repository
+
+find_git_repository_to_package_repository() {
+
+  local repository
+  local package_repository
+  local repo_path
+
+  package_repository="$1"
+
+  if [ "$1" = 'build-support' ]; then
+    echo 'packages'
+    return 0
+  fi
+
+  for repository in ${repo_names}; do
+    if [ "${repository}" = "archlinux32" ]; then
+      continue
+    fi
+    eval 'repo_path="${repo_paths__'"${repository}"'}"'
+    if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \
+      tar -t --wildcards '*/repos' | \
+      grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \
+      cut -d/ -f3 | \
+      sed 's|-[^-]\+$||' | \
+      sort -u | \
+      grep -qxF "${package_repository}"; then
+      echo "${repository}"
+      return 0
+    fi
+  done
+  >&2 echo "can't find git repository with package repository '$1'"
+  exit 1
+
+}
+
+# generate_package_metadata $package $git_revision $mod_git_revision $repository
+# or
+# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
+# generate the meta data files of a package (dependencies, built packages, ...)
+
+generate_package_metadata() {
+
+  local package="$1"
+  local git_revision="$2"
+  local mod_git_revision="$3"
+  local repository="$4"
+  local file_prefix
+  local file
+  local PKGBUILD
+
+  if [ $# -eq 1 ]; then
+    # second form
+    repository="${package##*.}"
+    package="${package%.*}"
+    mod_git_revision="${package##*.}"
+    package="${package%.*}"
+    git_revision="${package##*.}"
+    package="${package%.*}"
+  fi
+
+  file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
+
+  if [ -e "${file_prefix}.builds" ] && \
+    [ -e "${file_prefix}.build-depends" ] && \
+    [ -e "${file_prefix}.run-depends" ] && \
+    [ -e "${file_prefix}.groups" ] && \
+    [ -e "${file_prefix}.packages" ]; then
+    return 0
+  fi
+
+  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then
+    printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"
+    exit 1
+  fi
+  if [ ! -s "${file_prefix}.SRCINFO" ]; then
+    >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO"
+    exit 1
+  fi
+
+  # otherwise this just calls for trouble
+  sed -i '
+    /^[^=]*=\s*$/d
+    s/_i686\(\s*=\)/\1/
+  ' "${file_prefix}.SRCINFO"
+
+  # extract "groups" = groups \cup provides
+  grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \
+    cut -d= -f2 | \
+    sed 's|^\s\+||; s|[<>]$||' | \
+    sort -u > \
+    "${file_prefix}.groups"
+
+  # extract "packages" = pkgname
+  grep '^pkgname = ' "${file_prefix}.SRCINFO" | \
+    cut -d= -f2 | \
+    sed 's|^\s\+||; s|[<>]$||' | \
+    sort -u > \
+    "${file_prefix}.packages"
+
+  # extract "builds" = provides \cup pkgname \cup groups
+  cat "${file_prefix}.groups" "${file_prefix}.packages" | \
+    sort -u > \
+    "${file_prefix}.builds"
+
+  # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
+  {
+    {
+      printf 'all_depend = %s\n' 'base' 'base-devel'
+      sed -n "$(
+        printf '/^\t%s = /p\n' \
+          'depends' \
+          'makedepends'
+      )" "${file_prefix}.SRCINFO"
+    } | \
+      cut -d= -f2 | \
+      sed 's|^\s\+||; s|[<>]$||' | \
+      sort -u
+    sed 'p' "${file_prefix}.builds"
+  } | \
+    sort | \
+    uniq -u > \
+    "${file_prefix}.build-depends"
+
+  # extract "run-depends" = depends \cup \{ base \} \setminus "builds"
+  {
+    {
+      printf 'all_depend = %s\n' 'base'
+      sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO"
+    } | \
+      cut -d= -f2 | \
+      sed 's|^\s\+||; s|[<>]$||' | \
+      sort -u
+    sed 'p' "${file_prefix}.builds"
+  } | \
+    sort | \
+    uniq -u > \
+    "${file_prefix}.run-depends"
+
+  rm "${file_prefix}.SRCINFO"
+
+}
+
+# delete_old_metadata
+# delete old (=unneeded) meta data of packages
+
+delete_old_metadata() {
+
+  local current_metadata
+
+  current_metadata=$(
+    find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
+      sed '
+        s|\.[^.]\+$||
+        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
+      ' | \
+      sort -u
+  )
+
+  ( # the new shell is intentional
+    # what we have
+    echo "${current_metadata}"
+
+    # package-states should stay
+    find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
+      sed '
+        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
+      ' | \
+      sort -u | \
+      sed 'p'
+
+    # build-list items should stay
+    sed 'p' "${work_dir}/build-list"
+
+    tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir)
+    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+    echo "${current_metadata}" | \
+      sort -k1,1 > \
+      "${tmp_dir}/current-metadata"
+
+    # the newest of the following should stay:
+    {
+      # deletion-list items
+      cat "${work_dir}/deletion-list"
+      # all packages in the repos
+      for repo in ${repo_names}; do
+        eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \
+          tar -t | \
+          sed '
+            s|/$||
+            /\//d
+          '
+      done
+    } | \
+      sort -u | \
+      join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
+      sort -k4,4 > \
+      "${tmp_dir}/find-newest-revisions"
+
+    uniq -uf3 < \
+      "${tmp_dir}/find-newest-revisions" | \
+      awk '{print $4 " " $1 " " $2 " " $3}' | \
+      sed 'p'
+
+    uniq -Df3 < \
+      "${tmp_dir}/find-newest-revisions" | \
+      uniq --group=append -f3 | \
+      {
+        revs=''
+        mod_revs=''
+        opkg=''
+        orepo=''
+        while read -r rev mod_rev repo pkg; do
+
+          if [ -z "${rev}" ] && \
+            [ -z "${mod_rev}" ] && \
+            [ -z "${repo}" ] && \
+            [ -z "${pkg}" ]; then
+
+            printf '%s %s %s %s\n' \
+              "$(
+                printf '%s\n' ${revs} | \
+                  find_newest_of_git_revisions
+              )" \
+              "$(
+                printf '%s\n' ${mod_revs} | \
+                  find_newest_of_git_revisions
+              )" \
+              "${orepo}" \
+              "${opkg}"
+
+            revs=''
+            mod_revs=''
+            orepo=''
+            opkg=''
+            continue
+          fi
+          revs=$(
+            # shellcheck disable=SC2086
+            printf '%s\n' ${revs} ${rev} | \
+              sort -u
+          )
+          mod_revs=$(
+            # shellcheck disable=SC2086
+            printf '%s\n' ${mod_revs} ${mod_rev} | \
+              sort -u
+          )
+          orepo="${repo}"
+          opkg="${pkg}"
+        done
+      } | \
+      awk '{print $4 " " $1 " " $2 " " $3}' | \
+      sed 'p'
+  ) | \
+    sort | \
+    uniq -u | \
+    while read -r pkg rev mod_rev repo; do
+      rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
+    done
+}
+
+# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
+# print which (stable) repository a package belongs to
+
+repository_of_package() {
+  local package="$1"
+  local repository="${package##*.}"
+  package="${package%.*}"
+  local a32_rev="${package##*.}"
+  package="${package%.*.*}"
+
+  case "${repository}" in
+    'multilib')
+      if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \
+        tar -Ox | \
+        grep -qFx "${package%.*.*.*}"; then
+        echo 'extra'
+      else
+        echo 'community'
+      fi
+      ;;
+    *)
+      echo "${repository}"
+  esac
+}
+
+# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending
+# print wether the specified package is an official package (print
+# $ending) or a community package (print 'community-$ending') or a
+# build-support package (print 'build-support')
+
+official_or_community() {
+  local prepo
+  prepo=$(repository_of_package "$1")
+
+  if [ "${prepo}" = 'community' ]; then
+    echo 'community-'"$2"
+  elif [ "${prepo}" = 'build-support' ]; then
+    echo 'build-support'
+  else
+    echo "$2"
+  fi
+}
+
+# ls_master_mirror $path
+# list content of $path on the master mirror (via rsync)
+
+ls_master_mirror() {
+
+  local path="$1"
+
+  ${master_mirror_rsync_command} \
+    "${master_mirror_rsync_directory}/${path}/" | \
+    grep -v '\s\.$' | \
+    awk '{print $5}'
+
+}
+
+# TODO: the actions of remove_old_package_versions should be done
+# on basis of the information in the database
+
+# remove_old_package_versions $arch $repository $package_file
+
+# removes all older (not-newer) versions of $package_file
+# in all repositories not-older (newer) than $repository
+
+# TODO: should remove all other version (also newer) from
+# some repositories :-/
+
+# A package is considered not newer if
+#  a) its version is not newer
+# A package is considered older if
+#  b) its version is older or
+#  c) if it's "not newer" and its architecture is 'any' and different or
+#  d) if it's "not newer" and the other architecture is 'any' and different
+
+# this ensures an any package may replace arch-specific packages of the same version and vice versa
+
+remove_old_package_versions() {
+
+  local arch="$1"
+  local repository="$2"
+  local package="$3"
+
+  local pkgname
+  local epoch
+  local pkgver
+  local pkgrel
+  local sub_pkgrel
+  pkgname="${package%-*}"
+  pkgrel="${pkgname##*-}"
+  sub_pkgrel="${pkgrel##*.}"
+  if [ "${sub_pkgrel}" = "${pkgrel}" ]; then
+    sub_pkgrel='0'
+  else
+    pkgrel="${pkgrel%.*}"
+  fi
+  pkgname="${pkgname%-*}"
+  pkgver="${pkgname##*-}"
+  epoch="${pkgver%%:*}"
+  if [ "${epoch}" = "${pkgver}" ]; then
+    epoch='0'
+  else
+    pkgver="${pkgver#*:}"
+  fi
+  pkgname="${pkgname%-*}"
+
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \
+      "$(
+        printf '%s' "${repository}" | \
+          base64 -w0
+      )" \
+      "$(
+        printf '%s' "${package}" | \
+          base64 -w0
+      )" \
+      "$(
+        printf '%s' "${package}" | \
+          sed '
+            s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/
+          ' | \
+          base64 -w0
+      )"
+    printf 'SELECT '
+    printf '`binary_packages`.`id`,'
+    printf 'CONCAT(`repositories`.`name`,"/",'
+    mysql_package_name_query
+    printf '),'
+    # should we delete packages of identical version?
+    printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),'
+    printf 'CONCAT('
+      printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
+      printf '`binary_packages`.`pkgver`,"-",'
+      printf '`binary_packages`.`pkgrel`,".",'
+      printf '`binary_packages`.`sub_pkgrel`'
+    printf ')'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_binary_packages_architectures
+    printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
+    printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`'
+    # name must match
+    printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \
+      "$(printf '%s' "${package%-*-*-*}" | base64 -w0)"
+    # repository, where package should be deleted, should be less stable
+    printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \
+      "$(printf '%s' "${repository}" | base64 -w0)"
+    printf ';\n'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '
+      /^\S\+\sCONCAT(/d
+      y/\t/ /
+    ' | \
+    expand_version 4 | \
+    sort -k4V,4 -k3r,3 | \
+    shrink_version 4 | \
+    sed -n '
+      /^bogus /q
+      p
+    ' | \
+    cut -d' ' -f1,2 >&2
+
+  # repositories in which older packages should be deleted
+  local delete_older_repositories
+  # repositories in which not-newer packages should be deleted
+  local delete_not_newer_repositories
+
+  if echo "${standalone_package_repositories}" | \
+    grep -qxF "${repository}"; then
+
+    delete_older_repositories="${repository}"
+    delete_not_newer_repositories=''
+
+  elif echo "${staging_package_repositories}" | \
+    grep -qxF "${repository}"; then
+
+    delete_older_repositories="${repository}"
+    delete_not_newer_repositories=$(
+      echo "${staging_package_repositories}" | \
+        grep -vxF "${repository}"
+    ) || true
+
+  elif echo "${testing_package_repositories}" | \
+    grep -qxF "${repository}"; then
+
+    delete_older_repositories=$(
+      printf '%s\n' "${staging_package_repositories}" "${repository}"
+    )
+    delete_not_newer_repositories=$(
+      echo "${testing_package_repositories}" | \
+        grep -vxF "${repository}"
+    ) || true
+
+  elif echo "${stable_package_repositories}" | \
+    grep -qxF "${repository}"; then
+
+    delete_older_repositories=$(
+      printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}"
+    )
+    delete_not_newer_repositories=$(
+      echo "${stable_package_repositories}" | \
+        grep -vxF "${repository}"
+    ) || true
+
+  else
+
+    >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}"
+    return 1
+
+  fi
+
+  ( # the new shell is intentional
+    tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir)
+    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+    {
+      # the architecture of the package (any vs. i686)
+      package_arch="${package##*-}"
+      package_arch="${package_arch%%.*}"
+      if [ "${package_arch}" = 'any' ]; then
+        package_arch_regex_inverter='!'
+      else
+        unset package_arch_regex_inverter
+      fi
+
+      for repo in ${delete_older_repositories}; do
+        ls_master_mirror "${arch}/${repo}" | \
+          sed -n '
+            /\.pkg\.tar\.xz$/!d
+            s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
+            /^'"$(str_to_regex "${package%-*-*-*}")"' / {
+              s|^|2 '"${arch} ${repo}"' |
+              / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{
+                s|^2|0|
+              }
+              p
+            }
+          '
+      done
+      for repo in ${delete_not_newer_repositories}; do
+        ls_master_mirror "${arch}/${repo}" | \
+          sed -n '
+            /\.pkg\.tar\.xz$/!d
+            s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
+            /^'"$(str_to_regex "${package%-*-*-*}")"' / {
+              s|^|0 '"${arch} ${repo}"' |
+              p
+            }
+          '
+      done
+      echo "${package%-*}" | \
+        sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|'
+
+      # the generated list contains the following columns:
+      # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz
+    } | \
+      expand_version 5 | \
+      sort -k5V,5 -k1n,1 | \
+      shrink_version 5 | \
+      sed -n '
+        /^1 %cut% %it% %here% /q
+        s/^[02] //
+        s/ \(\S\+\)$/-\1/
+        p
+      ' | \
+      sort -u > \
+      "${tmp_dir}/packages-to-delete"
+    # this file contains a list of packages to be deleted, one on each line:
+    # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz
+
+    cut -d' ' -f1,2 < \
+      "${tmp_dir}/packages-to-delete" | \
+      grep -vxF "${arch} ${repository}" | \
+      sort -u > \
+      "${tmp_dir}/repositories-to-modify"
+
+    # fetch all databases being modified
+    while read -r del_arch del_repo; do
+      mkdir -p "${tmp_dir}/${del_arch}/${del_repo}"
+      ${master_mirror_rsync_command} \
+        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \
+        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \
+        "${tmp_dir}/${del_arch}/${del_repo}/"
+    done < \
+      "${tmp_dir}/repositories-to-modify"
+
+    while read -r del_arch del_repo del_package _; do
+      if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then
+        # we do not repo-remove the package in the target repository
+        continue
+      fi
+      repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \
+        "${del_package}"
+    done < \
+      "${tmp_dir}/packages-to-delete"
+
+    # upload modified databases
+    while read -r del_arch del_repo; do
+      ${master_mirror_rsync_command} \
+        "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \
+        "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \
+        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/"
+    done < \
+      "${tmp_dir}/repositories-to-modify"
+
+    # shellcheck disable=SC2016
+    sed '
+      s/\.pkg\.tar\.xz$//
+      s/^\S\+ //
+      s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/
+      s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/
+      s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/
+    ' "${tmp_dir}/packages-to-delete" | \
+      while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do
+        printf 'DELETE FROM `binary_packages` WHERE'
+        printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \
+          'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \
+          'repository'   'repositories'  'repositories'  'repositories'  "$(printf '%s' "${repo}" | base64 -w0)"
+        printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
+          'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \
+          'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
+          'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
+          'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
+          'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \
+          sed 's/ AND$//'
+        printf ';\n'
+      done | \
+      ${mysql_command}
+
+    sed '
+      s| \(\S\+\)$|-\1|
+      y| |/|
+      s|^|rm "|
+      s|$|"|
+      p
+      s|"$|.sig"|
+    ' "${tmp_dir}/packages-to-delete" | \
+      ${master_mirror_sftp_command}
+  )
+
+}
+
+# wait_some_time $minimum $diff
+# wait between minimum and minimum+diff seconds (diff defaults to 30)
+
+wait_some_time() {
+  local minimum=$1
+  local diff=$2
+  local random
+
+  if [ -z "${diff}" ]; then
+    diff=30
+  fi
+
+  random=$(
+    dd if='/dev/urandom' count=1 2> /dev/null | \
+      cksum | \
+      cut -d' ' -f1
+  )
+
+  sleep $((minimum + random % diff))
+}
+
+# str_to_regex $string
+# escape dots for use in regex
+
+str_to_regex() {
+  echo "$1" | \
+    sed '
+      s|[.[]|\\\0|g
+    '
+}
+
+# make_source_info $package $repository $git_revision $mod_git_revision $output
+# create .SRCINFO from PKGBUILD within git repositories, output to $output
+
+make_source_info() {
+
+  local package="$1"
+  local repository="$2"
+  local git_revision="$3"
+  local mod_git_revision="$4"
+  local output="$5"
+
+  local git_repo
+  local PKGBUILD
+  local PKGBUILD_mod
+
+  git_repo=$(find_repository_with_commit "${git_revision}")
+
+  if [ -z "${git_repo}" ]; then
+    return 1
+  fi
+
+  find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
+
+  ( # the new shell is intentional
+
+    tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX")
+    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+    extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0'
+
+    {
+      cd "${tmp_dir}"
+      makepkg --printsrcinfo
+      cd ..
+    } |
+    if [ "${package%-i18n}-i18n" = "${package}" ]; then
+      sed '
+        1 a \\tdepends = '"${package%-i18n}"'
+      '
+    else
+      cat
+    fi > \
+      "${output}"
+
+  )
+
+}
+
+# recursively_umount_and_rm $dir
+# umount all mountpoints in $dir which are also in $dir's
+#  filesystem, possibly also $dir itself and then
+#  rm -rf --one-file-system $dir
+
+recursively_umount_and_rm() {
+  local dir="$1"
+
+  if [ -z "${dir}" ]; then
+    >&2 echo 'ERROR: recursively_umount_and_rm requires an argument'
+    exit 42
+  fi
+
+  find "${dir}" \
+    -xdev -depth -type d \
+    -exec 'mountpoint' '-q' '{}' ';' \
+    -exec 'sudo' 'umount' '-l' '{}' ';'
+  rm -rf --one-file-system "${dir}"
+}
+
+# mangle_pkgbuild $PKGBUILD [$sub_pkgrel]
+#  mangle $arch in PKBUILDs to contain i486, i586, i686
+#  append $sub_pkgrel to the pkgrel
+
+mangle_pkgbuild() {
+  local PKGBUILD="$1"
+  local sub_pkgrel="$2"
+
+  if [ -n "${sub_pkgrel}" ]; then
+    sub_pkgrel=".${sub_pkgrel}"
+  fi
+
+  sed -i '
+    /^arch=[^#]*any/!{
+      /^arch=(/s/(/(i686 /
+    }
+    s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/
+  ' "${PKGBUILD}"
+}
+
+# find_newest_of_git_revisions
+# find newest git revision of the ones provided at stdin
+# (assuming linear history)
+
+find_newest_of_git_revisions() {
+  local revisions
+  local repo
+  revisions=$(cat)
+
+  if [ "$(
+    echo "${revisions}" | \
+      wc -l
+    )" -eq 1 ]; then
+
+    echo "${revisions}"
+    return
+
+  fi
+
+  repo=$(
+    find_repository_with_commit \
+      "$(
+        echo "${revisions}" | \
+          grep -xm1 '[0-9a-f]\{40\}'
+      )"
+  )
+
+  eval 'repo="${repo_paths__'"${repo}"'}"'
+
+  echo "${revisions}" | \
+    xargs -rn1 git -C "${repo}" rev-parse | \
+    {
+      newest=''
+      while read -r current; do
+        if [ -z "${newest}" ] || \
+          git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then
+            newest="${current}"
+          fi
+      done
+      echo "${newest}"
+    }
+}
+
+# find_package_repository_to_package $package $git_repository $git_commit
+# find the package repository a package from a given git repository
+# belongs to
+
+find_package_repository_to_package() {
+
+  local package="$1"
+  local git_repository="$2"
+  local git_commit="$3"
+  local repo_path
+  local repo
+
+  eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
+
+  if [ "${git_repository}" = 'archlinux32' ]; then
+    repo=$(
+      git -C "${repo_path}" archive "${git_commit}" -- | \
+        tar -t --wildcards "*/${package}/" | \
+        cut -d/ -f1 | \
+        sort -u
+    )
+  else
+    repo=$(
+      git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \
+        tar -t | \
+        cut -d/ -f3 | \
+        grep -vxF '' | \
+        grep -v 'staging\|testing\|-unstable' | \
+        grep -v -- '-i686$' | \
+        sed 's|-[^-]\+$||' | \
+        sort -u
+    )
+  fi
+
+  if [ -z "${repo}" ]; then
+    return 1
+  fi
+
+  if [ "$(
+    echo "${repo}" | \
+      wc -l
+    )" -ne 1 ]; then
+    return 1
+  fi
+
+  echo "${repo}"
+
+}
+
+# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel
+# extract files found in the svn/git source directories
+# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly
+
+extract_source_directory() {
+
+  local git_repo="$1"
+  local rev="$2"
+  local mod_rev="$3"
+  local output="$4"
+  local sub_pkgrel="$5"
+
+  if [ -n "${PKGBUILD}" ]; then
+    eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \
+      tar -x --strip-components=3 -C "${output}"
+  fi
+
+  if [ -n "${PKGBUILD_mod}" ]; then
+    git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \
+      tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \
+      true
+    git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \
+      tar -Ox "${PKGBUILD_mod}" >> \
+      "${output}/PKGBUILD"
+  fi
+
+  mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}"
+
+  # shellcheck disable=SC2016
+  sed -i '/^\$Id\$$/d' "${output}/PKGBUILD"
+
+}
+
+# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository
+# return a list of packages on the build list which are (run- / build- / check-time)
+# dependencies of the given package
+
+find_dependencies_on_build_list() {
+
+  local package="$1"
+  local git_revision="$2"
+  local mod_git_revision="$3"
+  local repository="$4"
+
+  generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
+
+  {
+    cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
+    awk '{print $1 "." $2 "." $3 "." $4}' < \
+      "${work_dir}/build-list" | \
+      sed '
+        s|^|'"${work_dir}"'/package-infos/|
+        s|$|\.builds|
+      ' | \
+      xargs -r cat | \
+      sort -u
+  } | \
+    sort | \
+    uniq -d
+
+}
+
+# download_sources_by_hash $package $repository $git_revision $git_mod_revision
+# try to download all sources by their hash into the current directory
+# returns 0 if any source was downloaded and 1 otherwise
+
+download_sources_by_hash() {
+
+  local package="$1"
+  local repository="$2"
+  local git_revision="$3"
+  local git_mod_revision="$4"
+
+  local return_value=1
+  local tmp_dir
+  local sum_type
+  local arch_suffix
+
+  tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir)
+
+  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then
+    >&2 echo 'download_sources_by_hash: make_source_info failed.'
+    rm -rf --one-file-system "${tmp_dir}"
+    return 1
+  fi
+
+  if ! [ -s "${tmp_dir}/.SRCINFO" ]; then
+    >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.'
+    rm -rf --one-file-system "${tmp_dir}"
+    return 1
+  fi
+
+  for arch_suffix in '' '_i686'; do
+    for sum_type in 'sha256sum' 'sha512sum'; do
+      grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
+        sed 's|^.* = ||' | \
+        cat -n > \
+        "${tmp_dir}/sums"
+      grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
+        sed '
+          s|^.* = ||
+          s|::.*$||
+          s|.*/||
+        ' | \
+        cat -n > \
+        "${tmp_dir}/urls"
+      if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then
+        join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \
+          "${tmp_dir}/joined"
+        while read -r sum file; do
+          if [ "${sum}" = 'SKIP' ]; then
+            continue
+          fi
+          if echo "${sum}  ${file}" | \
+            ${sum_type} -c > /dev/null 2>&1; then
+            # the correct source is already there
+            continue
+          fi
+          if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then
+            mv "${tmp_dir}/transfer" "${file}"
+            return_value=0
+          fi
+        done < \
+          "${tmp_dir}/joined"
+      fi
+    done
+  done
+
+  rm -rf --one-file-system "${tmp_dir}"
+  return ${return_value}
+
+}
+
+# expand_version $column_num
+# add "0:" to version in $colum_num-th column if no ":" is there (epoch)
+# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash)
+
+expand_version() {
+  local column_num
+  column_num="$1"
+
+  sed '
+    /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/
+    /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/
+  '
+}
+
+# shrink_version $column_num
+# remove "0:" from version in $colum_num-th column (epoch)
+# remove "+0" from version in $colum_num-th column (git count/hash)
+
+shrink_version() {
+  local column_num
+  column_num="$1"
+
+  sed '
+    s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/
+    s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/
+  '
+}
+
+# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ]
+
+# Return (to stdout) the biggest subset A of the packages in $omega whose
+# runtime dependencies in $omega \cup $keep are also in A
+
+# $all_builds and $all_depends either point to an empty file - then they will get
+# filled with cached data for subsequent calls - or to the same files of a previous
+# call
+
+# If non-empty, $force contains packages which are assumed to match the above
+# condition without checking.
+
+# The arguments are names of files with one $package.$revision.$mod_revision.$repository
+# per line.
+
+find_biggest_subset_of_packages() {
+
+  ( # the new shell is intentional
+    omega="$1"
+    keep="$2"
+    all_builds="$3"
+    all_depends="$4"
+    if [ $# -eq 4 ]; then
+      force='/dev/null'
+    elif [ $# -eq 5 ]; then
+      force="$5"
+    else
+      >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#"
+      return 2
+    fi
+
+    if [ ! -s "${all_builds}" ]; then
+      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \
+        -exec sed '
+          s|^|{} |
+          s|^\S\+/||
+          s|\.builds | |
+        ' {} \; | \
+        sort -k2,2 > \
+        "${all_builds}"
+    fi
+
+    if [ ! -s "${all_depends}" ]; then
+      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \
+        -exec sed '
+          s|^|{} |
+          s|^\S\+/||
+          s|\.run-depends | |
+        ' {} \; | \
+        grep -v ' base$' | \
+        sort -k2,2 > \
+        "${all_depends}"
+    fi
+
+    sort -u "${omega}" | \
+      sponge "${omega}"
+
+    temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir)
+    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
+
+    {
+      sort -u "${keep}"
+      cat "${force}" "${force}"
+    } | \
+      sort | \
+      uniq -u > \
+      "${temp_dir}/keep.new"
+    touch "${temp_dir}/keep"
+
+    while [ -s "${temp_dir}/keep.new" ]; do
+      cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \
+        sort -u | \
+        sponge "${temp_dir}/keep"
+
+      {
+        # we append all packages which are run-dependencies of keep-packages
+        # to the keep-list
+        sed '
+          s|^|'"${work_dir}"'/package-infos/|
+          s|$|.run-depends|
+        ' "${temp_dir}/keep" | \
+          xargs -r grep -HF '' | \
+          sed '
+            s|^.*/||
+            s|\.run-depends:| |
+          ' | \
+          sort -u | \
+          sort -k2,2 | \
+          uniq -f1 | \
+          join -1 2 -2 2 -o 2.1 - "${all_builds}"
+
+        # we append all packages with run-dependencies on the keep-list
+        # to the keep-list
+        sed '
+          s|^|'"${work_dir}"'/package-infos/|
+          s|$|.builds|
+        ' "${temp_dir}/keep" | \
+          xargs -r grep -HF '' | \
+          sed '
+            s|^.*/||
+            s|\.builds:| |
+          ' | \
+          sort -u | \
+          sort -k2,2 | \
+          uniq -f1 | \
+          join -1 2 -2 2 -o 2.1 - "${all_depends}"
+      } | \
+        sort -u | \
+        join -1 1 -2 1 -o 2.1 - "${omega}" | \
+        sort -u > \
+        "${temp_dir}/keep.new"
+
+      # "new" is only what has not been there before and what is not forced
+      cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \
+        sort | \
+        uniq -u | \
+        sponge "${temp_dir}/keep.new"
+    done
+
+    cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \
+      sort | \
+      uniq -u
+
+  )
+
+}
+
+# sort_square_bracket_content $file
+# sort the content of [] in $file, print to stdout
+
+sort_square_bracket_content() {
+  local file
+  local line
+  local token
+  local token_list
+  local rest
+  file="$1"
+
+  while read -r line; do
+    printf '%s ' "${line}" | \
+      tr ' ' '\n' | \
+      while read -r token; do
+        if echo "${token}" | \
+          grep -qF '['; then
+          printf '%s[' "${token%[*}"
+          token="${token##*[}"
+          token_list="${token%,}"
+          while ! echo "${token_list}" | \
+            grep -qF ']'; do
+            read -r token
+            token_list=$(
+              printf '%s\n' \
+                "${token_list}" \
+                "${token%,}"
+            )
+          done
+          rest="]${token_list#*]}"
+          token_list="${token_list%%]*}"
+          token=$(
+            printf '%s' "${token_list}" | \
+              sort | \
+              sed 's|$|,|'
+            printf '%s' "${rest}"
+          )
+        fi
+        printf '%s\n' "${token}"
+      done | \
+      tr '\n' ' ' | \
+      sed '
+        s|, ]|]|g
+        s| $||
+      '
+    printf '\n'
+  done < \
+    "${file}"
+}
+
+# smoothen_namcap_log $file
+# remove unneccesary differences from namcap-logs:
+#   - remove architecture specific information
+#   - sort lines
+#   - sort content of square brackets
+
+smoothen_namcap_log() {
+  local file
+  file="$1"
+  # shellcheck disable=SC2016
+  sort_square_bracket_content "${file}" | \
+    sed '
+      # normalize architecture specific information
+      s|i[34567]86|$ARCH|g
+      s|x86\([-_]64\)\?|$ARCH|g
+      # remove haskell hashes
+      s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g
+    ' | \
+    sort | \
+    sponge "${file}"
+}
+
+# print_list_of_archaic_packages $source1 $source2 ...
+# print a list of packages which have not been touched for a while,
+# but which are still in the pipeline, e.g. in $source1, $source2 or ...
+
+print_list_of_archaic_packages() {
+  for source in "$@"; do
+    case "${source}" in
+      'testing')
+        # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32
+        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \
+          -exec head -n1 {} \; | \
+          "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin
+        # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated
+        # and no longer block other packages from being moved
+        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \
+          sed '
+            s|\.testing$||
+          '
+      ;;
+      'build-list')
+        while read -r pkg rev mod_rev repo; do
+          git_repo=$(
+            find_repository_with_commit "${rev}"
+          )
+          eval repo_path='"${repo_paths__'"${git_repo}"'}"'
+          commit_date=$(
+            git -C "${repo_path}" show -s --format=%ct "${rev}"
+          )
+          mod_commit_date=$(
+            git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}"
+          )
+          if [ "${mod_commit_date}" -gt "${commit_date}" ]; then
+            commit_date="${mod_commit_date}"
+          fi
+          # packages remaining longer than $max_package_age_build_list days on the build list
+          if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then
+            printf '%s %s %s %s\n' \
+              "${pkg}" \
+              "${rev}" \
+              "${mod_rev}" \
+              "${repo}"
+          fi
+        done < \
+          "${work_dir}/build-list"
+      ;;
+      'staging')
+        # packages remaining longer than $max_package_age_staging days in staging
+        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \
+          sed '
+            s|\.done$||
+          '
+      ;;
+      *)
+        >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}"
+      ;;
+    esac
+  done | \
+    sort -u
+}
+
+# modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
+# print the given modification revision possibly with a html link to github
+
+modification_revision_link() {
+  local mod_rev="$1"
+  local repo="$2"
+  local pkg="$3"
+
+  if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then
+    printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \
+      "${mod_rev}" \
+      "${repo}" \
+      "${pkg}" \
+      "${mod_rev}"
+  else
+    printf '%s\n' \
+      "${mod_rev}"
+  fi
+}
+
+# trigger_mirror_refreshs
+# trigger a refresh of capable tier 1 mirrors (as backup for master mirror)
+
+trigger_mirror_refreshs() {
+  local tmp_file
+
+  tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir)
+  date '+%s' > \
+    "${tmp_file}"
+  ${master_mirror_rsync_command} \
+    "${tmp_file}" \
+    "${master_mirror_rsync_directory}/lastupdate"
+  rm "${tmp_file}"
+  for trigger_url in ${mirror_refresh_trigger_urls}; do
+    screen -S trigger-mirror-update -d -m curl -L "${trigger_url}"
+  done
+}
+
+# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name
+extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() {
+  pkgname="$1"
+  pkgname="${pkgname%.pkg.tar.xz}"
+  arch="${pkgname##*-}"
+  pkgname="${pkgname%-*}"
+  sub_pkgrel="${pkgname##*-}"
+  pkgname="${pkgname%-*}"
+  pkgrel="${sub_pkgrel%.*}"
+  if [ "${pkgrel}" = "${sub_pkgrel}" ]; then
+    sub_pkgrel='0'
+  else
+    sub_pkgrel="${sub_pkgrel##*.}"
+  fi
+  epoch="${pkgname##*-}"
+  pkgname="${pkgname%-*}"
+  pkgver="${epoch#*:}"
+  if [ "${pkgver}" = "${epoch}" ]; then
+    epoch='0'
+  else
+    epoch="${epoch%%:*}"
+  fi
+}
diff --git a/lib/mysql-functions b/lib/mysql-functions
new file mode 100755
index 0000000..185d9a9
--- /dev/null
+++ b/lib/mysql-functions
@@ -0,0 +1,1098 @@
+#!/bin/sh
+
+# contains functions used to access mysql db
+
+# shellcheck disable=SC2016,SC2039
+
+if [ -z "${base_dir}" ]; then
+  # just to make shellcheck happy
+  . 'conf/default.conf'
+fi
+
+# base64_encode_each encode each line of stdin with base64
+
+base64_encode_each() {
+  local line
+
+  while read -r line; do
+    printf '%s' \
+      "${line}" | \
+      base64 -w0
+    printf '\n'
+  done
+}
+
+# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
+
+# shellcheck disable=SC2086
+mysql_add_package_source() {
+  local names='pkgbase git_revision mod_git_revision upstream_package_repository'
+  local values
+  local uses_upstream
+  local uses_modification
+  local repo
+
+  if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then
+    uses_modification=1
+  else
+    uses_modification=0
+  fi
+  uses_upstream=0
+  for repo in ${repo_names}; do
+    if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
+      tar -t 2>/dev/null | \
+      sed 's,-x86_64/,-any/,' | \
+      grep -qFx "$1/repos/$4-any/PKGBUILD"; then
+      uses_upstream=1
+    fi
+  done
+
+  for _ in ${names}; do
+    values="${values}$(
+      printf '%s' "$1" | \
+        base64 -w0
+    ) "
+    shift
+  done
+  values="${values% }"
+
+  {
+    printf 'INSERT IGNORE INTO package_sources'
+    printf ' ('
+    printf '`%s`, ' ${names}
+    printf '`uses_upstream`,`uses_modification`'
+    printf ') SELECT'
+    printf ' from_base64("%s"), ' ${values% *}
+    printf ' `upstream_repositories`.`id`,%s,%s' \
+      ${uses_upstream} ${uses_modification}
+    printf ' FROM `upstream_repositories`'
+    printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \
+      "${values##* }"
+  } | \
+    ${mysql_command}
+}
+
+# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository
+
+# shellcheck disable=SC2031,SC2086,SC2154
+mysql_add_binary_package() {
+  local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository'
+  local name
+  for name in ${names}; do
+    eval 'local '"${name}"
+    eval "${name}"'=$(
+      printf "%s" "$1" |
+        base64 -w0
+    )'
+    shift
+  done
+
+  {
+    printf 'INSERT IGNORE INTO binary_packages'
+    printf ' ('
+    printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted'
+    printf ') SELECT'
+    printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}"
+    printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures'
+    printf ' 0, 0, 0'
+    printf ' FROM'
+    printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures'
+    printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
+    printf ' WHERE'
+    printf ' `%s`.`name` = from_base64("%s") AND' \
+      'repositories' "${repository}" \
+      'architectures' "${architecture}"
+    printf ' `package_sources`.`%s` = from_base64("%s") AND' \
+      'pkgbase' "${pkgbase}" \
+      'git_revision' "${git_revision}" \
+      'mod_git_revision' "${mod_git_revision}"
+    printf ' `upstream_repositories`.`name` = from_base64("%s")' \
+      "${upstream_package_repository}"
+  } | \
+    sed '
+      s|, )|)|g
+      s|, FROM| FROM|g
+    ' | \
+    ${mysql_command}
+}
+
+# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel
+
+# shellcheck disable=SC2031,SC2086,SC2154
+mysql_show_binary_package() {
+  local names='pkgname pkgver pkgrel sub_pkgrel'
+  local name
+  for name in ${names}; do
+    eval 'local '"${name}"
+    eval "${name}"'=$(
+      printf "%s" "$1" |
+        base64 -w0
+    )'
+    shift
+  done
+
+  {
+    printf 'SELECT'
+    printf ' `%s`.`%s`,' \
+      'repositories' 'name' \
+      'binary_packages' 'pkgname' \
+      'package_sources' 'pkgver' \
+      'package_sources' 'pkgrel' \
+      'binary_packages' 'sub_pkgrel' \
+      'architectures' 'name' \
+      'package_sources' 'pkgbase' \
+      'package_sources' 'git_revision' \
+      'package_sources' 'mod_git_revision' \
+      'upstream_repositories' 'name'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packages_architectures
+    mysql_join_binary_packages_repositories
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
+    printf ' WHERE'
+    printf ' `%s`.`%s` = from_base64("%s") AND' \
+      'binary_packages' 'pkgname' "${pkgname}" \
+      'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \
+      'package_sources' 'pkgver' "${pkgver}" \
+      'package_sources' 'pkgrel' "${pkgrel}"
+    printf ';'
+  } | \
+    sed '
+      s|, FROM| FROM|g
+      s|AND;|;|g
+    ' | \
+    ${mysql_command} --html
+}
+
+# mysql_add_install_target $install_target
+
+# shellcheck disable=2086
+mysql_add_install_target() {
+  local install_target
+  install_target=$(
+    printf "%s" "$1" | \
+      base64 -w0
+  )
+
+  {
+    printf 'INSERT IGNORE INTO `install_targets` (`name`)'
+    printf ' VALUES (from_base64("%s"))' \
+      "${install_target}"
+  } | \
+    ${mysql_command}
+}
+
+# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository
+# or
+# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository
+# if sub_pkgrel should be determined automatically
+# and
+# mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository
+# or
+# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository
+# if $sub_pkgrel should be forced
+
+# generate the meta data of a package (dependencies, built packages, ...) in the database
+
+mysql_generate_package_metadata() {
+
+  ( # new shell is intentional
+    case "$1" in
+      ''|*[!0-9]*)
+        unset forced_sub_pkgrel
+      ;;
+      *)
+        forced_sub_pkgrel=$(
+          printf '%s' "$1" | \
+            base64 -w0
+        )
+        shift
+      ;;
+    esac
+    current_repository="$1"
+    package="$2"
+
+    if [ $# -eq 2 ]; then
+      # second form
+      repository="${package##*.}"
+      package="${package%.*}"
+      mod_git_revision="${package##*.}"
+      package="${package%.*}"
+      git_revision="${package##*.}"
+      package="${package%.*}"
+    else
+      git_revision="$3"
+      mod_git_revision="$4"
+      repository="$5"
+    fi
+
+    temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir)
+    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
+
+    printf '.' >&2
+    if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then
+      printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"
+      exit 2
+    fi
+    # remove empty lines and unsupported architectures
+    sed -i '
+      /^[^=]*=\s*$/d
+      /^\s*arch = /{
+        / \(i686\|any\)$/!d
+      }
+    ' "${temp_dir}/SRCINFO"
+
+    if [ ! -s "${temp_dir}/SRCINFO" ]; then
+      >&2 printf '"make_source_info" had empty output - eh, what?\n'
+      exit 2
+    fi
+    printf '\n\n' >> "${temp_dir}/SRCINFO"
+
+    printf '.' >&2
+    pkgbase=$(
+      grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \
+        cut -d' ' -f3
+    )
+    if [ -z "${pkgbase}" ]; then
+      >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n'
+      exit 2
+    fi
+
+    # add the package source
+    mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
+    printf '.' >&2
+
+    # now we encode everything in base64
+    current_repository=$(
+      printf '%s' "${current_repository}" | \
+        base64 -w0
+    )
+    pkgbase=$(
+      printf '%s' "${pkgbase}" | \
+        base64 -w0
+    )
+    git_revision=$(
+      printf '%s' "${git_revision}" | \
+        base64 -w0
+    )
+    mod_git_revision=$(
+      printf '%s' "${mod_git_revision}" | \
+        base64 -w0
+    )
+    repository=$(
+      printf '%s' "${repository}" | \
+        base64 -w0
+    )
+
+    # add the build assignment(s)
+    {
+      archs=$(
+        sed -n '
+          s/^\tarch = //
+          T
+          p
+        ' "${temp_dir}/SRCINFO" | \
+          grep -vxF 'any' | \
+          sort -u
+      )
+      if [ -z "${archs}" ]; then
+        echo 'any'
+      else
+        printf '%s\n' "${archs}"
+      fi
+    } | \
+      while read -r arch; do
+        printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
+        printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0'
+        printf ' FROM `architectures` JOIN `package_sources`'
+        printf ' WHERE `architectures`.`name` = from_base64("%s")' \
+          "$(
+            printf '%s' "${arch}" | \
+              base64 -w0
+          )"
+        printf ' AND `package_sources`.`%s` = from_base64("%s")' \
+          'pkgbase' "${pkgbase}" \
+          'git_revision' "${git_revision}" \
+          'mod_git_revision' "${mod_git_revision}"
+        printf ';\n'
+      done > \
+      "${temp_dir}/add-build-assignments-command"
+
+    # TODO: correctly link between binary_packages and build_assignments using any_arch
+
+    # shellcheck disable=SC2034
+    # select any specific arch (which will be building the 'any' part of a split package)
+    any_arch=$(
+      {
+        sed -n '
+          s/^\tarch = //
+          T
+          p
+        ' "${temp_dir}/SRCINFO" | \
+          sort -r | \
+          grep -vxFm 1 'any' || \
+          echo 'any'
+      } | \
+        base64_encode_each
+    )
+
+    grep '^pkgname = ' "${temp_dir}/SRCINFO" | \
+      cut -d' ' -f3 | \
+      while read -r pkgname; do
+        pkgname64=$(
+          printf '%s' "${pkgname}" | \
+            base64 -w0
+        )
+        sed -n '
+          /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{
+            :a
+              N
+              /\n$/{
+                p
+                T
+              }
+              ba
+          }
+        ' "${temp_dir}/SRCINFO" | \
+          sed '
+            /^\S/d
+            s/^\s*//
+          ' > \
+          "${temp_dir}/BINARYINFO.${pkgname64}"
+
+        grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \
+          cut -d' ' -f3 | \
+          while read -r arch; do
+            arch64=$(
+              printf '%s' "${arch}" | \
+                base64 -w0
+            )
+            sed '
+              s/^\(\S\+\)_'"${arch}"' = /\1 = /
+            ' "${temp_dir}/BINARYINFO.${pkgname64}" > \
+              "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}"
+          done
+      done
+    find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \
+      while read -r _ pkgname arch; do
+        pkgver=$(
+          grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+            cut -d' ' -f3 | \
+            base64_encode_each
+        )
+        pkgrel=$(
+          grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+            cut -d' ' -f3 | \
+            base64_encode_each
+        )
+        epoch=$(
+          {
+            grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \
+              echo 'epoch = 0'
+          } | \
+            cut -d' ' -f3 | \
+            base64_encode_each
+        )
+        provides=$(
+          grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+            cut -d' ' -f3 | \
+            sed 's/[<>=].*$//' | \
+            base64_encode_each
+        )
+        makedepends=$(
+          grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+            cut -d' ' -f3 | \
+            sed 's/[<>=].*$//' | \
+            base64_encode_each
+        )
+        checkdepends=$(
+          grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+            cut -d' ' -f3 | \
+            sed 's/[<>=].*$//' | \
+            base64_encode_each
+        )
+        rundepends=$(
+          grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
+            cut -d' ' -f3 | \
+            sed 's/[<>=].*$//' | \
+            base64_encode_each
+        )
+        if [ -n "${forced_sub_pkgrel}" ]; then
+          sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")'
+        else
+          sub_pkgrel=$(
+            printf '(SELECT COALESCE('
+            # do not add binary packages which are currently on the
+            # build-list or in $current_repository (beware of split
+            # packages!)
+            printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
+            mysql_join_binary_packages_architectures
+            mysql_join_binary_packages_repositories
+            printf ' WHERE'
+            printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
+              'epoch' "${epoch}" \
+              'pkgver' "${pkgver}" \
+              'pkgrel' "${pkgrel}" \
+              'pkgname' "${pkgname}"
+            printf ' `architectures`.`name`=from_base64("%s")' \
+              "${arch}"
+            printf ' AND `repositories`.`name` IN ("build-list",from_base64("%s"))),' \
+              "${current_repository}"
+            # max(sub_pkgrel)+1
+            printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
+            mysql_join_binary_packages_architectures
+            printf ' WHERE'
+            printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
+              'epoch' "${epoch}" \
+              'pkgver' "${pkgver}" \
+              'pkgrel' "${pkgrel}" \
+              'pkgname' "${pkgname}"
+            if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then
+              # 'any' gets higher sub_pkgrel than any architecture
+              printf ' 1'
+            else
+              # not-'any' gets higher sub_pkgrel than same or 'any' architecture
+              printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \
+                "${arch}"
+            fi
+            printf ')'
+            printf ',0))'
+          )
+        fi
+        {
+          printf 'INSERT IGNORE INTO `binary_packages` ('
+          printf '`%s`,' \
+            'build_assignment' \
+            'repository' \
+            'architecture' \
+            'epoch' \
+            'pkgver' \
+            'pkgrel' \
+            'pkgname' \
+            'sub_pkgrel' \
+            'has_issues' \
+            'is_tested' \
+            'is_to_be_deleted'
+          printf ') SELECT '
+          printf '`%s`.`id`,' \
+            'build_assignments' \
+            'repositories' \
+            'architectures'
+          printf 'from_base64("%s"),' \
+            "${epoch}" \
+            "${pkgver}" \
+            "${pkgrel}" \
+            "${pkgname}"
+          printf '%s,0,0,0 FROM' \
+            "${sub_pkgrel}"
+          printf ' `%s` JOIN' \
+            'repositories' \
+            'architectures' \
+            'build_assignments'
+          mysql_join_build_assignments_package_sources
+          mysql_join_package_sources_upstream_repositories
+          printf ' WHERE'
+          printf ' `%s`.`%s` = from_base64("%s") AND' \
+            'repositories' 'name' "${current_repository}" \
+            'architectures' 'name' "${arch}" \
+            'package_sources' 'pkgbase' "${pkgbase}" \
+            'package_sources' 'git_revision' "${git_revision}" \
+            'package_sources' 'mod_git_revision' "${mod_git_revision}" \
+            'upstream_repositories' 'name' "${repository}"
+          printf ';\n'
+        } | \
+          sed '
+            s|,)|)|g
+            s| JOIN JOIN | JOIN |
+            s| AND;$|;|
+          ' >> \
+          "${temp_dir}/add-binary-packages-command"
+        {
+          printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \
+            'provides' \
+            'makedepends' \
+            'checkdepends' \
+            'rundepends'
+
+          printf 'INSERT INTO `provides` VALUES\n'
+          echo "${provides}" | \
+            sort -u | \
+            grep -vxF '' | \
+            sed '
+              s|^| (from_base64("|
+              s|$|")),|
+            '
+          printf ' (from_base64("%s"));\n' \
+            "${pkgname}"
+
+          printf 'INSERT INTO `rundepends` VALUES\n'
+          echo "${rundepends}" | \
+            sort -u | \
+            grep -vxF '' | \
+            sed '
+              s|^| (from_base64("|
+              s|$|")),|
+            '
+          printf ' ("base");\n'
+
+          echo "${checkdepends}" | \
+            sort -u | \
+            grep -vxF '' | \
+            sed '
+              s|^| (from_base64("|
+              s|$|")),|
+              1 s/^/INSERT INTO `checkdepends` VALUES \n/
+              $ s/,$/;/
+            '
+
+          printf 'INSERT INTO `makedepends` VALUES\n'
+          echo "${makedepends}" | \
+            sort -u | \
+            grep -vxF '' | \
+            sed '
+              s|^| (from_base64("|
+              s|$|")),|
+            '
+          printf ' ("base-devel");\n'
+
+          printf 'INSERT IGNORE INTO `install_targets` (`name`)'
+          printf ' SELECT (`name`) FROM `%s` UNION' \
+            'provides' \
+            'makedepends' \
+            'checkdepends' \
+            'rundepends' | \
+            sed 's| UNION$|;\n|'
+
+          for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
+            case "${link}" in
+              'provides')
+                printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT'
+                printf ' `binary_packages`.`id`,`install_targets`.`id` FROM'
+              ;;
+              'makedepends'|'checkdepends'|'rundepends')
+                printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT'
+                printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM'
+                printf ' `dependency_types` JOIN'
+              ;;
+            esac
+            printf ' `binary_packages`'
+            mysql_join_binary_packages_architectures
+            printf ' JOIN `install_targets`'
+            printf ' JOIN `%s`' "${link}"
+            printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
+            printf ' WHERE'
+            if [ "${link}" = 'makedepends' ] || \
+              [ "${link}" = 'checkdepends' ] || \
+              [ "${link}" = 'rundepends' ]; then
+              printf ' `dependency_types`.`name` = "%s" AND' \
+                "${link%depends}"
+            fi
+            printf ' `binary_packages`.`%s` = from_base64("%s") AND' \
+              'epoch' "${epoch}" \
+              'pkgver' "${pkgver}" \
+              'pkgrel' "${pkgrel}" \
+              'pkgname' "${pkgname}"
+            # we do not want to match the sub_pkgrel:
+            # a) it is tedious to do so (because it may be calculated
+            #    dynamically)
+            # b) it is not necessary to do so: if only the sub_pkgrel
+            #    changed, the dependencies and provided install_targets
+            #    should not have changed
+            printf ' `architectures`.`name` = from_base64("%s");\n' \
+              "${arch}"
+            # the repository is of no relevance: it hardly matters for
+            # the dependencies
+          done
+
+          printf 'DROP TABLE `%s`;\n' \
+            'provides' \
+            'makedepends' \
+            'checkdepends' \
+            'rundepends'
+        } >> \
+          "${temp_dir}/add-install-targets-command"
+      done
+    printf '.' >&2
+
+    {
+      if [ -s "${temp_dir}/add-build-assignments-command" ]; then
+        cat "${temp_dir}/add-build-assignments-command"
+      fi
+      if [ -s "${temp_dir}/add-binary-packages-command" ]; then
+        cat "${temp_dir}/add-binary-packages-command"
+      fi
+      if [ -s "${temp_dir}/add-install-targets-command" ]; then
+        cat "${temp_dir}/add-install-targets-command"
+      fi
+    } | \
+      ${mysql_command}
+    printf '.' >&2
+
+  )
+}
+
+# mysql_sanity_check
+# do a sanity check on the mysql database
+
+mysql_sanity_check() {
+  {
+    printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packages_build_assignments
+    mysql_join_binary_packages_architectures '' 'bp_arch'
+    mysql_join_build_assignments_architectures '' 'ba_arch'
+    printf ' WHERE `bp_arch`.`name`!="any"'
+    printf ' AND `ba_arch`.`name`="any";\n'
+    printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)'
+    printf ' FROM `binary_packages` AS `a`'
+    printf ' JOIN `binary_packages` AS `b`'
+    printf ' ON `a`.`pkgname`=`b`.`pkgname`'
+    printf ' AND `a`.`repository`=`b`.`repository`'
+    printf ' AND `a`.`id`!=`b`.`id`'
+    mysql_join_binary_packages_repositories 'a'
+    printf ' WHERE `repositories`.`name`="build-list";\n'
+    printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)'
+    printf ' FROM `binary_packages` AS `a`'
+    printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`'
+    mysql_join_binary_packages_repositories 'a' 'arep'
+    mysql_join_binary_packages_repositories 'b' 'brep'
+    printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`'
+    printf ' AND `%srep`.`name`="build-list"' \
+      'a' 'b'
+    printf ';\n'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '
+      /^CONCAT("/d
+      s,^,<font color="#FF0000">,
+      s,$,</font>,
+    '
+  ( # new shell is intentional
+    temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir)
+    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
+
+    for dir in $(ls_master_mirror 'i686'); do
+      ls_master_mirror "i686/${dir}" | \
+        sed -n '
+          s/\.pkg\.tar\.xz$//
+          T
+          s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/
+          s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/
+          s|^|'"${dir}"'/|
+          p
+        '
+    done | \
+      sort > \
+      "${temp_dir}/master-mirror-listing"
+
+    {
+      printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`'
+      printf ' FROM `binary_packages`'
+      mysql_join_binary_packages_architectures
+      mysql_join_binary_packages_repositories
+      printf ' WHERE `repositories`.`is_on_master_mirror`'
+    } | \
+      ${mysql_command} --batch | \
+      sed '
+        1d
+        s,\t,/,
+        s,\t,-,
+        s,\t,:,
+        s,\t,-,
+        s,\t,.,
+        s,\t,-,
+      ' | \
+      sort > \
+      "${temp_dir}/mysql-packages"
+
+    diff -u \
+      "${temp_dir}/master-mirror-listing" \
+      "${temp_dir}/mysql-packages"
+  )
+}
+
+mysql_find_build_assignment_loops() {
+  new_loops=$(
+    {
+      printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`'
+      printf ' FROM `dependencies`'
+      mysql_join_dependencies_install_target_providers
+      mysql_join_install_target_providers_binary_packages '' 'packages_dependency'
+      mysql_join_dependencies_binary_packages '' 'packages_dependent'
+      mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency'
+      mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent'
+      printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
+    } | \
+      ${mysql_command} --raw --batch | \
+      sed '
+        1d
+        y/\t/ /
+      ' | \
+      tsort 2>&1 >/dev/null | \
+      sed 's/^tsort:\s*//' | \
+      {
+        loop=0
+        while read -r id; do
+          if [ "x${id}" = 'x-: input contains a loop:' ]; then
+            loop=$((loop+1))
+            continue
+          fi
+          if ! printf '%s' "${id}" | tr '\n' ' ' | grep -q '^[0-9]\+$'; then
+            >&2 printf 'ERROR: non-numeric id "%s"\n' "${id}"
+            continue
+          fi
+          printf '(%s,%s),' "${loop}" "${id}"
+        done | \
+          sed 's/,$//'
+      }
+  )
+  {
+    printf 'DELETE FROM `build_dependency_loops`;\n'
+    if [ -n "${new_loops}" ]; then
+      printf 'INSERT INTO `build_dependency_loops` (`loop`,`build_assignment`) VALUES %s;\n' \
+        "${new_loops}"
+    fi
+  } | \
+    ${mysql_command}
+}
+
+# mysql_cleanup [dry]
+# clean up left overs from mysql database
+mysql_cleanup() {
+  local operator
+  if [ "$#" = '0' ]; then
+    operator='DELETE'
+  elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then
+    operator='SELECT COUNT(*)'
+  else
+    >&2 echo 'Unknown parameter'
+    >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".'
+    exit 2
+  fi
+  {
+    # remove to-be-decided binary_packages
+    printf '%s ' \
+      "${operator}"
+    if [ "${operator}" = 'DELETE' ]; then
+      printf '`binary_packages` '
+    fi
+    printf 'FROM `binary_packages`'
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="to-be-decided";\n'
+    # remove dependencies w/o binary_package or install_target
+    printf '%s FROM `dependencies` ' \
+      "${operator}"
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `binary_packages` '
+      printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`'
+    printf ') OR NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `install_targets` '
+      printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`'
+    printf ');\n'
+    # remove install_target_providers w/o binary_package or install_target
+    printf '%s FROM `install_target_providers` ' \
+      "${operator}"
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `binary_packages` '
+      printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`'
+    printf ') OR NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `install_targets` '
+      printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`'
+    printf ');\n'
+    # remove build_assignments w/o binary_package
+    printf '%s FROM `build_assignments` ' \
+      "${operator}"
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `binary_packages` '
+      printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+    printf ');\n'
+    # remove failed_builds with unbroken build_assignments
+    printf '%s ' \
+      "${operator}"
+    if [ "${operator}" = 'DELETE' ]; then
+      printf '`failed_builds` '
+    fi
+    printf 'FROM `failed_builds` '
+    mysql_join_failed_builds_build_assignments
+    printf 'WHERE NOT `build_assignments`.`is_broken`'
+    printf ';\n'
+    # remove failed_builds w/o build_assignment
+    printf '%s FROM `failed_builds` ' \
+      "${operator}"
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `build_assignments` '
+      printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`'
+    printf ');\n'
+    # remove package_sources w/o build_assignment
+    printf '%s FROM `package_sources` ' \
+      "${operator}"
+    printf 'WHERE NOT EXISTS '
+    printf '('
+      printf 'SELECT * FROM `build_assignments` '
+      printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`'
+    printf ');\n'
+    printf 'UPDATE `build_slaves`'
+    mysql_join_build_slaves_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' SET `currently_building`=NULL'
+    printf ' WHERE `repositories`.`name`!="build-list";\n'
+  } | \
+    ${mysql_command}
+}
+
+# mysql_query_has_pending_dependencies `build_assignment`.`id`
+# print a mysql query giving wether dependencies are pending
+mysql_query_has_pending_dependencies() {
+  printf 'EXISTS ('
+    printf 'SELECT * FROM `binary_packages` as `to_dos`'
+    mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos'
+    mysql_join_binary_packages_dependencies 'to_dos'
+    mysql_join_dependencies_install_target_providers
+    mysql_join_install_target_providers_binary_packages '' 'bin_deps'
+    mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos'
+    printf ' WHERE'
+    printf ' `%s`.`name`="build-list" AND' \
+      'dep_repos' 'to_do_repos'
+    printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND'
+    printf ' `to_dos`.`build_assignment`=%s' \
+      "$1"
+  printf ')'
+}
+
+# mysql_query_is_part_of_loop `build_assignment`.`id`
+# print a mysql query giving wether the package is part of a loop
+mysql_query_is_part_of_loop() {
+  printf 'EXISTS ('
+    printf 'SELECT * FROM `build_dependency_loops`'
+    printf ' WHERE `build_dependency_loops`.`build_assignment`=%s' \
+      "$1"
+  printf ')'
+}
+
+# mysql_query_select_pkgbase_and_revision
+# print the part of a mysql query giving:
+# pkgbase git_revision mod_git_revision upstream_package_repository
+mysql_query_select_pkgbase_and_revision() {
+  printf '`package_sources`.`%s`,' \
+    'pkgbase' \
+    'git_revision' \
+    'mod_git_revision'
+  printf '`upstream_repositories`.`name`'
+  printf ' FROM `build_assignments`'
+  mysql_join_build_assignments_package_sources
+  mysql_join_package_sources_upstream_repositories
+}
+
+# mysql_repair_binary_packages_without_build_assignment
+# try to generate valid build assignments to binary packages without
+# a valid one yet
+mysql_repair_binary_packages_without_build_assignment() {
+  {
+    printf 'SELECT '
+    printf '`binary_packages`.`id`'
+    printf ',replace(to_base64(%s),"\\n","")' \
+      '`binary_packages`.`pkgname`' \
+      '`architectures`.`name`'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packages_architectures
+    printf ' WHERE `binary_packages`.`build_assignment`<0'
+  } | \
+    ${mysql_command} --raw --batch | \
+    sed '1d' | \
+    while read -r id pkgname arch; do
+      pkgname=$(
+        printf '%s' "${pkgname}" | \
+          base64 -d
+      )
+      pkgbase=$(
+        curl -Ss "$(
+          printf 'https://www.archlinux.org/packages/search/json/?name=%s' \
+            "${pkgname}"
+        )" | \
+          sed '
+            s/^.*"results":\s*\[//
+            s/}\s*,\s*{/\n/g
+          ' | \
+          grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \
+          tr ',' '\n' | \
+          grep '"pkgbase":' | \
+          cut -d'"' -f4 | \
+          sort -u | \
+          head -n1
+      )
+      if [ -z "${pkgbase}" ] && \
+        {
+          printf 'SELECT count(*) FROM `package_sources`'
+          printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
+            "$(printf '%s' "${pkgname}" | base64 -w0)"
+        } | \
+          ${mysql_command} --raw --batch | \
+          sed '1d' | \
+          grep -qvxF '0'; then
+        pkgbase="${pkgname}"
+      fi
+      if [ -z "${pkgbase}" ]; then
+        >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}"
+        continue
+      fi
+      pkgbase=$(
+        printf '%s' "${pkgbase}" | \
+          base64 -w0
+      )
+      printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
+      printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0'
+      printf ' FROM `package_sources`'
+      printf ' JOIN `architectures`'
+      printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}"
+      printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}"
+      printf ' LIMIT 1;\n'
+      printf 'UPDATE `binary_packages`'
+      printf ' JOIN `build_assignments`'
+      mysql_join_binary_packages_build_assignments
+      printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+      printf ' WHERE `binary_packages`.`id`=%s' "${id}"
+      printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}"
+    done | \
+    ${mysql_command}
+}
+
+# mysql_remove_duplicate_binary_packages
+# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel,
+# having difference of 1 in sub_pkgrel
+
+mysql_remove_duplicate_build_order() {
+  {
+    printf 'CREATE TEMPORARY TABLE `ren`'
+    printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n'
+    printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)'
+    printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`'
+    printf ' FROM `binary_packages` as `old`'
+    printf ' JOIN `binary_packages` as `new` ON'
+    printf ' `old`.`%s`=`new`.`%s` AND' \
+      'pkgname' 'pkgname' \
+      'epoch' 'epoch' \
+      'pkgver' 'pkgver' \
+      'pkgrel' 'pkgrel'
+    printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`'
+    mysql_join_binary_packages_repositories 'old' 'orep'
+    mysql_join_binary_packages_repositories 'new' 'nrep'
+    printf ' WHERE `orep`.`name`!="to-be-decided"'
+    printf ' AND `nrep`.`name`="to-be-decided";\n'
+    printf 'UPDATE IGNORE `dependencies`'
+    printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`'
+    printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n'
+    printf 'UPDATE IGNORE `install_target_providers`'
+    printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`'
+    printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n'
+    printf 'DELETE FROM `binary_packages`'
+    printf ' WHERE EXISTS ('
+      printf 'SELECT * FROM `ren`'
+      printf ' WHERE `ren`.`old`=`binary_packages`.`id`'
+    printf ');\n'
+    printf 'UPDATE IGNORE `binary_packages`'
+    printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`'
+    printf ' SET `binary_packages`.`repository`=`ren`.`repo`,'
+    printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n'
+  } | \
+    ${mysql_command}
+}
+
+# mysql_package_name_query
+# print a mysql query of the full name of a package file
+
+mysql_package_name_query() {
+  printf 'CONCAT('
+  printf '`binary_packages`.`pkgname`,"-",'
+  printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
+  printf '`binary_packages`.`pkgver`,"-",'
+  printf '`binary_packages`.`pkgrel`,".",'
+  printf '`binary_packages`.`sub_pkgrel`,"-",'
+  printf '`architectures`.`name`,".pkg.tar.xz"'
+  printf ')'
+}
+
+# mysql_join_*_*
+# print 'JOIN' part of mysql query to connect the respective tables
+# these functions take 2 optional arguments, acting as aliases for
+# the tables
+
+# mysql_join__generic $table_a $column_a $table_b $column_b
+# create mysql_join_${table_a}_${table_b}() function
+
+mysql_join__generic() {
+  eval "$(
+    printf 'mysql_join_%s_%s() {\n' "$1" "$3"
+    printf '  printf '"'"' JOIN `%s`'"'"'\n' "$3"
+    printf '  if [ -n "$2" ]; then\n'
+    printf '    printf '"'"' AS `%%s`'"'"' "$2"\n'
+    printf '  fi\n'
+    printf '  if [ -n "$1" ]; then\n'
+    printf '    printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2"
+    printf '  else\n'
+    printf '    printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2"
+    printf '  fi\n'
+    printf '  if [ -n "$2" ]; then\n'
+    printf '    printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4"
+    printf '  else\n'
+    printf '    printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4"
+    printf '  fi\n'
+    printf '}\n'
+  )"
+}
+
+for link in \
+  'binary_packages:architecture:architectures' \
+  'binary_packages:repository:repositories' \
+  'binary_packages:build_assignment:build_assignments' \
+ \
+  'build_assignments:architecture:architectures' \
+  'build_assignments:package_source:package_sources' \
+ \
+  'build_dependency_loops:build_assignment:build_assignments' \
+  'build_dependency_loops:build_assignment build_assignment:binary_packages' \
+ \
+  'build_slaves:currently_building:build_assignments' \
+  'build_slaves:currently_building build_assignment:binary_packages' \
+ \
+  'dependencies:depending_on:install_targets' \
+  'dependencies:dependent:binary_packages' \
+  'dependencies:dependency_type:dependency_types' \
+ \
+  'failed_builds:reason:fail_reason' \
+  'failed_builds:build_assignment:build_assignments' \
+  'failed_builds:build_slave:build_slaves' \
+ \
+  'install_target_providers:package:binary_packages' \
+  'install_target_providers:install_target:install_targets' \
+  'install_target_providers:install_target depending_on:dependencies' \
+ \
+  'package_sources:upstream_package_repository:upstream_repositories' \
+ \
+  'repositories:stability:repository_stabilities' \
+ \
+  'upstream_repositories:git_repository:git_repositories'; do
+# A join for these cannot be done, because it's not clear on what to join:
+#  'repository_stability_relations:more_stable:repository_stabilities'
+#  'repository_stability_relations:less_stable:repository_stabilities'
+
+  table_b="${link##*:}"
+  table_a="${link%:*}"
+  column_b="${table_a##*:}"
+  table_a="${table_a%:*}"
+  column_a="${column_b% *}"
+  if [ "${column_a}" = "${column_b}" ]; then
+    column_b='id'
+  else
+    column_b="${column_b##* }"
+  fi
+
+  mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}"
+  mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}"
+done
-- 
cgit v1.2.3-54-g00ecf


From d9a9ab193a00123d5839505ddfe1563c2286bb9f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 24 Feb 2018 19:48:14 +0100
Subject: bin/strict-bashism-check: check lib/, too

---
 bin/strict-bashism-check | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/strict-bashism-check b/bin/strict-bashism-check
index 3bb3884..b1bd3bf 100755
--- a/bin/strict-bashism-check
+++ b/bin/strict-bashism-check
@@ -52,7 +52,7 @@ git archive "${tree}" | \
 errors=$(
   cd "${tmp_dir}" || \
     echo 'Cannot cd.'
-  find "bin" "conf" -type f -not -executable -not -name '.gitignore'
+  find bin conf lib -type f -not -executable -not -name '.gitignore'
 )
 
 if [ -n "${errors}" ]; then
@@ -63,7 +63,7 @@ fi
 
 errors=$(
 # shellcheck disable=SC2016
-  find bin conf -type f -executable -exec grep -H '="\$(' {} \;
+  find bin conf lib -type f -executable -exec grep -H '="\$(' {} \;
 )
 
 if [ -n "${errors}" ]; then
-- 
cgit v1.2.3-54-g00ecf


From 45e7f08ab436087414ba10c862d5f8a59b516d03 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 09:00:53 +0100
Subject: lib/common-functions: save log of executed scripts - for debugging
 mysterious errors :-)

---
 conf/default.conf    | 4 ++--
 lib/common-functions | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/conf/default.conf b/conf/default.conf
index 18f5d73..7e7ed3d 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -14,13 +14,13 @@ else
   base_dir=$(printf '%s/..' "$(dirname "$(readlink -f "$0")")")
 fi
 
+work_dir="${base_dir}/work"
+
 # shellcheck source=lib/common-functions
 . "${base_dir}/lib/common-functions"
 # shellcheck source=lib/mysql-functions
 . "${base_dir}/lib/mysql-functions"
 
-work_dir="${base_dir}/work"
-
 repo_names='packages community archlinux32'
 repo_paths__packages="${work_dir}/repos/packages"
 repo_paths__community="${work_dir}/repos/community"
diff --git a/lib/common-functions b/lib/common-functions
index a84271b..80f252c 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -15,6 +15,15 @@ if [ -z "${base_dir}" ]; then
   . 'conf/default.conf'
 fi
 
+{
+  printf 'sourcing common-functions for %s\n' "$0"
+  printf '%s parameters:' "$#"
+  printf ' "%s"' "$@"
+  printf '\n'
+} >> \
+  "${work_dir}/command-log"
+
+
 # find_pkgbuilds package repository git_repository git_revision mod_git_revision
 # find the PKGBUILD and modification of $package from $repository
 # sets $PKGBUILD and $PKGBUILD_mod
-- 
cgit v1.2.3-54-g00ecf


From 8fae4bb4650ea62def1c6f90bfa8fd89b2caabde Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 09:50:56 +0100
Subject: bin/bootstrap-mysql: simplify

---
 bin/bootstrap-mysql | 33 +++++++++------------------------
 1 file changed, 9 insertions(+), 24 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 4dee0f8..48f2850 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -139,6 +139,7 @@ if [ ! "$1" = 'slim' ]; then
       ' less_stable MEDIUMINT :repository_stabilities'
   )
 
+  # shellcheck disable=SC2016
   {
     printf '%s\n' \
       '/*!40014 SET UNIQUE_CHECKS=0 */;' \
@@ -151,14 +152,11 @@ if [ ! "$1" = 'slim' ]; then
         while read -r table size; do
           case "${turn}" in
            'drop')
-              # shellcheck disable=SC2016
               printf 'DROP TABLE IF EXISTS `%s`;\n' "${table}"
             ;;
             'create'|'link')
               if [ "${turn}" = 'create' ]; then
-                # shellcheck disable=SC2016
                 printf 'CREATE TABLE `%s` (\n' "${table}"
-                # shellcheck disable=SC2016
                 printf '  `id` %s NOT NULL AUTO_INCREMENT,\n' "${size}"
               fi
               echo "${tables}" | \
@@ -181,16 +179,14 @@ if [ ! "$1" = 'slim' ]; then
                       if [ -z "${rest}" ]; then
                         rest="${type}"
                       fi
-                      # shellcheck disable=SC2016
                       printf '  UNIQUE KEY `%s` (' "${type}"
-                      # shellcheck disable=SC2016,SC2086
+                      # shellcheck disable=SC2086
                       printf '`%s`,' ${rest} | \
                         sed 's|,$||'
                       printf ')'
                     ;;
                     *)
                       if [ "${turn}" = 'create' ]; then
-                        # shellcheck disable=SC2016
                         printf '  `%s` %s' "${column}" "${type}"
                         if ! echo "${rest}" | \
                           grep -qwF 'NULL'; then
@@ -201,7 +197,6 @@ if [ ! "$1" = 'slim' ]; then
                           tr ' ' '\n' | \
                           sed -n 's/^://;T;p' | \
                           while read -r link; do
-                            # shellcheck disable=SC2016
                             printf 'ALTER TABLE `%s` ADD FOREIGN KEY (`%s`) REFERENCES `%s` (`id`);\n' \
                               "${table}" \
                               "${column}" \
@@ -215,7 +210,6 @@ if [ ! "$1" = 'slim' ]; then
                   fi
                 done
               if [ "${turn}" = 'create' ]; then
-                # shellcheck disable=SC2016
                 printf '  PRIMARY KEY (`id`));\n'
               fi
             ;;
@@ -230,26 +224,26 @@ if [ ! "$1" = 'slim' ]; then
       '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;'
     printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \
       'CREATE TEMPORARY TABLES' 'buildmaster.*' \
-      'SHOW VIEW' 'buildmaster.*' \
+      'EXECUTE' 'buildmaster.*' \
+      'RELOAD' '*.*' \
       'SELECT' 'buildmaster.*' \
-      'UPDATE' 'buildmaster.*' \
-      'RELOAD' '*.*'
-    printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \
       'SHOW VIEW' 'buildmaster.*' \
-      'SELECT' 'buildmaster.*'
+      'UPDATE' 'buildmaster.*'
+    printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \
+      'SELECT' 'buildmaster.*' \
+      'SHOW VIEW' 'buildmaster.*'
     printf 'FLUSH PRIVILEGES;\n'
   } | \
     mysql -u root -p buildmaster
 fi
 
+# shellcheck disable=SC2016
 {
-  # shellcheck disable=SC2016
   printf 'INSERT IGNORE INTO `architectures` (`name`) VALUES '
   printf '("%s"),' \
     'any' 'i686' | \
     sed 's|,$||'
   printf ';\n'
-  # shellcheck disable=SC2016
   printf 'INSERT IGNORE INTO `fail_reasons` (`identifier`,`name`,`severity`) VALUES '
   printf '(from_base64("%s"),"%s",%s),' \
     "$(
@@ -290,7 +284,6 @@ fi
     )" 'unknown' 100 | \
     sed 's|,$||'
   printf ';\n'
-  # shellcheck disable=SC2016
   printf 'INSERT IGNORE INTO `git_repositories` (`name`,`url`,`directory`,`head`) VALUES'
   {
     for repo in ${repo_names}; do
@@ -328,9 +321,7 @@ fi
     'extra:packages' \
     'multilib:packages' \
     'community:community'; do
-    # shellcheck disable=SC2016
     printf 'INSERT IGNORE INTO `upstream_repositories` (`name`,`git_repository`) SELECT\n'
-    # shellcheck disable=SC2016
     printf '  from_base64("%s"),`id` FROM `git_repositories` WHERE `name` = from_base64("%s");\n' \
       "$(
         printf '%s' "${repo%:*}" | \
@@ -342,7 +333,6 @@ fi
       )"
   done
 
-  # shellcheck disable=SC2016
   printf 'INSERT IGNORE INTO `build_slaves` (`name`,`ssh_key`,`operator`,`last_connection`) VALUES'
   {
     sed -n '
@@ -385,7 +375,6 @@ fi
   } | \
     sed 's|,;|;|'
 
-  # shellcheck disable=SC2016
   printf 'INSERT IGNORE INTO `repository_stabilities` (`name`,`bugtracker_category`) VALUES'
   {
     printf '\n  ("%s",%s),' \
@@ -399,7 +388,6 @@ fi
   } | \
     sed 's|,;|;|'
 
-  # shellcheck disable=SC2016
   {
     printf 'INSERT IGNORE INTO `repository_stability_relations` (`more_stable`,`less_stable`)'
     printf ' SELECT `ms`.`id`,`ls`.`id`'
@@ -435,9 +423,7 @@ fi
     'community-staging:staging:AQ==' \
     'build-list:unbuilt:AA==' \
     'deletion-list:forbidden:AA=='; do
-    # shellcheck disable=SC2016
     printf 'INSERT IGNORE INTO `repositories` (`name`,`stability`,`is_on_master_mirror`) SELECT'
-    # shellcheck disable=SC2016
     printf ' from_base64("%s"),`id`,from_base64("%s") FROM `repository_stabilities` WHERE `name`=from_base64("%s");\n' \
       "$(
         printf '%s' "${repo}" | \
@@ -455,7 +441,6 @@ fi
       )"
   done
 
-  # shellcheck disable=SC2016
   printf 'INSERT IGNORE INTO `dependency_types` (`name`,`relevant_for_building`,`relevant_for_binary_packages`) VALUES'
   {
     printf '\n  ("%s",%s,%s),' \
-- 
cgit v1.2.3-54-g00ecf


From 2e3532e6a9e9b69b722b1a8da51fd2d27e895887 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 09:54:40 +0100
Subject: bin/bootstrap-mysql: show_broken_packages_and_dependencies new

---
 bin/bootstrap-mysql | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 48f2850..fd670c7 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -222,7 +222,45 @@ if [ ! "$1" = 'slim' ]; then
     printf '%s\n' \
       '/*!40014 SET UNIQUE_CHECKS=1 */;' \
       '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;'
+
+    printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n'
+    printf 'DELIMITER //\n'
+    printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n'
+    printf 'BEGIN\n'
+    printf 'CREATE TEMPORARY TABLE `%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \
+      'broken_packages_and_dependencies' 'broken_packages_and_dependencies_old'
+    printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)'
+    printf ' SELECT `build_assignments`.`id`'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packages_build_assignments
+    printf ' AND `build_assignments`.`is_broken`'
+    mysql_join_binary_packages_repositories
+    printf ' AND `repositories`.`name`="build-list";\n'
+    printf 'REPEAT\n'
+    printf 'INSERT IGNORE INTO `broken_packages_and_dependencies_old` (`id`)'
+    printf ' SELECT `broken_packages_and_dependencies`.`id` FROM `broken_packages_and_dependencies`;\n'
+    printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)'
+    printf ' SELECT `new_bp`.`build_assignment`'
+    printf ' FROM `broken_packages_and_dependencies_old`'
+    mysql_join_build_assignments_binary_packages 'broken_packages_and_dependencies_old' 'old_bp'
+    mysql_join_binary_packages_dependencies 'old_bp'
+    mysql_join_dependencies_dependency_types
+    printf ' AND `dependency_types`.`relevant_for_building`'
+    mysql_join_dependencies_install_target_providers
+    mysql_join_install_target_providers_binary_packages '' 'new_bp'
+    mysql_join_binary_packages_repositories 'new_bp' 'new_repo'
+    printf ' AND `new_repo`.`name` IN ("build-list","deletion-list");\n'
+    printf 'UNTIL ROW_COUNT()=0\n'
+    printf 'END REPEAT;\n'
+    printf 'SELECT '
+    mysql_query_select_pkgbase_and_revision
+    printf ' JOIN `broken_packages_and_dependencies` ON `broken_packages_and_dependencies`.`id`=`build_assignments`.`id`;\n'
+    printf 'END\n'
+    printf '//\n'
+    printf 'DELIMITER ;\n'
+
     printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \
+      'CREATE ROUTINE' 'buildmaster.*' \
       'CREATE TEMPORARY TABLES' 'buildmaster.*' \
       'EXECUTE' 'buildmaster.*' \
       'RELOAD' '*.*' \
-- 
cgit v1.2.3-54-g00ecf


From e6957b88ae30b32305167e8784f6b31eefd80895 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 09:56:34 +0100
Subject: bin/show-dependencies: use show_broken_packages_and_dependencies

---
 bin/show-dependencies | 93 +++------------------------------------------------
 1 file changed, 5 insertions(+), 88 deletions(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index bb1ae37..ec19ac8 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -9,95 +9,12 @@
 
 if [ $# -eq 0 ]; then
   broken=$(
-    # shellcheck disable=SC2016
-    {
-      printf 'SELECT `package_sources`.`pkgbase`'
-      printf ' FROM `package_sources`'
-      mysql_join_package_sources_build_assignments
-      printf ' WHERE `build_assignments`.`is_broken`'
-    } | \
+    printf 'CALL show_broken_packages_and_dependencies;\n' | \
       ${mysql_command} --raw --batch | \
-      sed '1d' | \
-      sort -u
-  )
-  new_sum='x'
-  sum=''
-  tmp_dir=$(mktemp -d 'tmp.show-dependencies.0.XXXXXXXXXX' --tmpdir)
-  trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`binary_packages`.`pkgname`'
-    printf ' FROM `binary_packages`'
-    mysql_join_binary_packages_build_assignments
-    mysql_join_build_assignments_package_sources
-    printf ' ORDER BY `binary_packages`.`pkgname`'
-    # Why can't mysql order in the same way "sort" does!
-  } | \
-    ${mysql_command} --raw --batch | \
-    sed '
-      1d
-      y/\t/ /
-    ' | \
-    sort -k2,2 > \
-    "${tmp_dir}/builds"
-  while ! [ "${new_sum}" = "${sum}" ]; do
-    sum="${new_sum}"
-    # shellcheck disable=SC2086
-    printf '%s\n' ${broken} | \
-      sort > \
-      "${tmp_dir}/broken"
-    broken=$(
-      {
-        find "${work_dir}/package-infos" -maxdepth 1 -name '*.build-depends' | \
-          sed 's|^.*/\(.*\)\(\.[^.]\+\)\{4\}$|\1 \0|' | \
-          sort -k1,1 | \
-          join -1 1 -2 1 -o 1.2 - "${tmp_dir}/broken" | \
-          xargs -r cat
-      } | \
-        sort -u | \
-        join -1 1 -2 2 -o 2.1 - "${tmp_dir}/builds"
-    )
-    broken=$(
-      {
-        {
-          # shellcheck disable=SC2086
-          printf '%s\n' ${broken}
-          cat "${tmp_dir}/broken"
-        } | \
-          sort -u
-        cut -d' ' -f1 < \
-          "${work_dir}/build-list" | \
-          sort -u
-      } | \
-        sort | \
-        uniq -d
-    )
-    new_sum=$(
-      # shellcheck disable=SC2086
-      printf '%s\n' ${broken} | \
-        sha512sum
-    )
-  done
-
-  rm -rf --one-file-system "${tmp_dir}"
-  trap - EXIT
-  broken=$(
-    {
-      # shellcheck disable=SC2086
-      printf '%s\n' ${broken} 'ALL'
-      {
-        find "${webserver_directory}/graphs" -maxdepth 1 -name '*.png' -printf '%f\n' | \
-          sed 's|\.png$||'
-        {
-          cut -d' ' -f1 < \
-            "${work_dir}/build-list"
-          cat "${work_dir}/deletion-list"
-        } | \
-          sort -u
-      } | \
-        sort | \
-        uniq -d
-    } | \
+      sed '
+        1d
+        s/\s.*$//
+      ' | \
       sort -u
   )
   # shellcheck disable=SC2086
-- 
cgit v1.2.3-54-g00ecf


From 8a13eca562eec640a9974e32ca5798b0c59cd99e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 10:00:27 +0100
Subject: bin/show-dependencies: forgot "ALL"

---
 bin/show-dependencies | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index ec19ac8..030beab 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -18,7 +18,7 @@ if [ $# -eq 0 ]; then
       sort -u
   )
   # shellcheck disable=SC2086
-  "$0" ${broken}
+  "$0" ALL ${broken}
   {
     find "${webserver_directory}/graphs" -maxdepth 1 -name '*.png'
     # shellcheck disable=SC2086
-- 
cgit v1.2.3-54-g00ecf


From b2a175e38818f4bdf088d2695e243572fe5eb2e8 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 10:26:33 +0100
Subject: bin/bootstrap-mysql: add comment

---
 bin/bootstrap-mysql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index fd670c7..275764b 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -223,6 +223,9 @@ if [ ! "$1" = 'slim' ]; then
       '/*!40014 SET UNIQUE_CHECKS=1 */;' \
       '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;'
 
+    # show_broken_packages_and_dependencies
+    # gives a list of broken packages and their dependencies if they're
+    # still on the build- or deletion-list
     printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n'
     printf 'DELIMITER //\n'
     printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n'
-- 
cgit v1.2.3-54-g00ecf


From 0ed8a1ed58785d3b0fcb21cf3f911d3d1111424a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 10:36:51 +0100
Subject: bin/show-dependencies: _really_ plot ALL-graph

---
 bin/show-dependencies | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index 030beab..95b596a 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -16,9 +16,10 @@ if [ $# -eq 0 ]; then
         s/\s.*$//
       ' | \
       sort -u
+    printf 'ALL\n'
   )
   # shellcheck disable=SC2086
-  "$0" ALL ${broken}
+  "$0" ${broken}
   {
     find "${webserver_directory}/graphs" -maxdepth 1 -name '*.png'
     # shellcheck disable=SC2086
-- 
cgit v1.2.3-54-g00ecf


From 5a72e06a2ef65e8cf2991938896bc520ae2b5c4a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 15:20:54 +0100
Subject: bin/bootstrap-mysql:
 calculate_dependencies_of_package_upto_first_built_one new

---
 bin/bootstrap-mysql | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 275764b..a172d82 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -262,6 +262,40 @@ if [ ! "$1" = 'slim' ]; then
     printf '//\n'
     printf 'DELIMITER ;\n'
 
+    # calculate_dependencies_of_package_upto_first_built_one
+    # save binary packages (only their `id`) in `relevant_binary_packages` and `relevant_binary_packages_copy`
+    printf 'DROP PROCEDURE IF EXISTS calculate_dependencies_of_package_upto_first_built_one;\n'
+    printf 'DELIMITER //\n'
+    printf 'CREATE PROCEDURE calculate_dependencies_of_package_upto_first_built_one(IN `target_pkgbase` VARCHAR(64))\n'
+    printf 'BEGIN\n'
+    printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)'
+    printf ' SELECT `binary_packages`.`id`'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packages_repositories
+    printf ' AND `repositories`.`name`="build-list"'
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
+    printf ' WHERE `package_sources`.`pkgbase`=`target_pkgbase`;\n'
+    printf 'REPEAT\n'
+    printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)'
+    printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`;\n'
+    printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)'
+    printf ' SELECT `install_target_providers`.`package`'
+    printf ' FROM `relevant_binary_packages_copy`'
+    printf ' JOIN `binary_packages` ON `relevant_binary_packages_copy`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_repositories
+    printf ' AND `repositories`.`name`="build-list"'
+    mysql_join_binary_packages_dependencies
+    mysql_join_dependencies_dependency_types
+    printf ' AND `dependency_types`.`relevant_for_building`'
+    mysql_join_dependencies_install_target_providers
+    printf ';\n'
+    printf 'UNTIL ROW_COUNT()=0\n'
+    printf 'END REPEAT;\n'
+    printf 'END\n'
+    printf '//\n'
+    printf 'DELIMITER ;\n'
+
     printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \
       'CREATE ROUTINE' 'buildmaster.*' \
       'CREATE TEMPORARY TABLES' 'buildmaster.*' \
-- 
cgit v1.2.3-54-g00ecf


From 399476dfe28c050114420fdadcf9f97c8e2ce384 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 15:23:19 +0100
Subject: bin/show-dependencies: read information from database

---
 bin/show-dependencies | 563 ++++++++++++++++++++------------------------------
 1 file changed, 224 insertions(+), 339 deletions(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index 95b596a..bce97d1 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -3,7 +3,7 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: read information from database
+# TODO: can this be faster?
 
 # Create a lock file.
 
@@ -54,360 +54,249 @@ fi
 tmp_dir=$(mktemp -d 'tmp.show-dependencies.1.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-sort -u "${work_dir}/build-order" | \
-  grep -v '^\(\S\+\) \1$' | \
-  grep '^\S\+ \S\+$' > \
-  "${tmp_dir}/original-build-order"
-
-sort -k1,1 "${work_dir}/build-list" > \
-  "${tmp_dir}/build-list.sorted-by-package"
-
-{
-  sort -k1,1 "${tmp_dir}/original-build-order" | \
-    join -1 1 -2 1 -o 1.1,1.2 - "${tmp_dir}/build-list.sorted-by-package"
-  sort -k2,2 "${tmp_dir}/original-build-order" | \
-    join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/build-list.sorted-by-package"
-} | \
-  sort -u | \
-  sponge "${tmp_dir}/original-build-order"
-
-find "${work_dir}/package-infos" -maxdepth 1 -printf '%f\n' | \
-  sed 's|\(\.[^.]\+\)\{4\}$||' | \
-  sort -u > \
-  "${tmp_dir}/known-packages"
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.groups' -printf '%f %p\n' | \
-  while read -r name pf; do
-    xargs -r printf "${name%.*.*.*.*}"' %s\n' < \
-      "${pf}"
-  done > \
-  "${tmp_dir}/known-groups"
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f %p\n' | \
-  while read -r name pf; do
-    grep -vxF "${name%.*.*.*.*}" "${pf}" | \
-      xargs -r printf "${name%.*.*.*.*}"' %s\n'
-  done > \
-  "${tmp_dir}/known-split-packages"
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.run-depends' -printf '%f %p\n' | \
-  while read -r name pf; do
-    xargs -r printf '%s '"${name%.*.*.*.*}"'\n' < \
-      "${pf}"
-  done > \
-  "${tmp_dir}/known-run-depends"
-
 for target_package in "$@"; do
 
-  output="${webserver_directory}/graphs/${target_package}.png"
-
-  if [ "${target_package}" = 'ALL' ]; then
-
-    # groups and split packages built by jobs on the build list
-    cat "${tmp_dir}/known-groups" "${tmp_dir}/known-split-packages" | \
-      sort -k1,1 | \
-      join -1 1 -2 1 -o 1.2 - "${tmp_dir}/build-list.sorted-by-package" | \
-      sort -u > \
-      "${tmp_dir}/relevant-stuff"
-
-    {
-      # groups and split packages built by jobs on the build list - which are also dependencies of packages on the build-list
-      sort -k1,1 "${tmp_dir}/original-build-order" | \
-        join -1 1 -2 1 -o 2.1 - "${tmp_dir}/relevant-stuff"
-      # build list jobs themself
-      cut -d' ' -f1 < "${work_dir}/build-list"
-    } | \
-      sort -u | \
-      sponge "${tmp_dir}/relevant-stuff"
-
-    {
-      sort -k1,1 "${tmp_dir}/original-build-order" | \
-        join -1 1 -2 1 -o 1.1,1.2 - "${tmp_dir}/relevant-stuff" | \
-        sort -k2,2 | \
-        join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/relevant-stuff"
-      sed 's/.*/\0 \0/' "${tmp_dir}/relevant-stuff"
-    } > \
-      "${tmp_dir}/build-order"
-
-    rm "${tmp_dir}/relevant-stuff"
-
-    find "${work_dir}/package-states" -name '*.done' -printf '%f\n' | \
-      sed 's|\(\.[^.]\+\)\{4\}$||' | \
-      sort -u | \
-      tee "${tmp_dir}/done-packages" | \
-      xargs -r printf '4 staging-package %s\n' > \
-      "${tmp_dir}/knots"
-
-    {
-      cat \
-        "${tmp_dir}/known-groups" \
-        "${tmp_dir}/known-run-depends" \
-        "${tmp_dir}/known-split-packages"
-      awk '{print $1 " " $1}' < \
-        "${tmp_dir}/known-packages"
-    } | \
-      sort -k1,1 > \
-      "${tmp_dir}/known-connections.sorted.1"
-
-    sort -k2,2 \
-      "${tmp_dir}/known-connections.sorted.1" > \
-      "${tmp_dir}/known-connections.sorted.2"
-
-    {
-      # staging -> something -> build-list
-      join -1 1 -2 1 -o 1.1,1.2 \
-        "${tmp_dir}/known-connections.sorted.1" \
-        "${tmp_dir}/done-packages" | \
-        sort -k2,2 | \
-        join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \
-        - \
-        "${tmp_dir}/known-connections.sorted.1" | \
-        sort -k4,4 | \
-        join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \
-        - \
-        "${tmp_dir}/build-list.sorted-by-package" | \
-        sed '
-          s|^\(\S\+ \S\+\) |\1\n|
-        '
-      # staging -> something -> staging
-      join -1 1 -2 1 -o 1.1,1.2 \
-        "${tmp_dir}/known-connections.sorted.1" \
-        "${tmp_dir}/done-packages" | \
-        sort -k2,2 | \
-        join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \
-        - \
-        "${tmp_dir}/known-connections.sorted.1" | \
-        sort -k4,4 | \
-        join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \
-        - \
-        "${tmp_dir}/done-packages" | \
-        sed '
-          s|^\(\S\+ \S\+\) |\1\n|
-        '
-      # build-list -> something -> staging
-      join -1 1 -2 1 -o 1.1,1.2 \
-        "${tmp_dir}/known-connections.sorted.1" \
-        "${tmp_dir}/build-list.sorted-by-package" | \
-        sort -k2,2 | \
-        join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \
-        - \
-        "${tmp_dir}/known-connections.sorted.1" | \
-        sort -k4,4 | \
-        join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \
-        - \
-        "${tmp_dir}/done-packages" | \
-        sed '
-          s|^\(\S\+ \S\+\) |\1\n|
-        '
-      cat "${tmp_dir}/build-order"
-    } | \
-      sort -u | \
-      sponge "${tmp_dir}/build-order"
-
-    rm \
-      "${tmp_dir}/done-packages" \
-      "${tmp_dir}/known-connections.sorted.1" \
-      "${tmp_dir}/known-connections.sorted.2"
-
-  else
-
-    grep " $(str_to_regex "${target_package}")\$" "${tmp_dir}/original-build-order" | \
-      sort -u > \
-      "${tmp_dir}/build-order"
-    printf '0 target-package %s\n' "${target_package}" > \
-      "${tmp_dir}/knots"
-
-    last_sum=''
-    current_sum=$(sha512sum "${tmp_dir}/build-order")
-    while ! [ "${last_sum}" = "${current_sum}" ]; do
-
-      last_sum="${current_sum}"
-
-      awk '{print $1}' "${tmp_dir}/build-order" | \
-        sort -u > \
-        "${tmp_dir}/new"
-      {
-        cat \
-          "${tmp_dir}/original-build-order" \
-          "${tmp_dir}/known-split-packages"
-        sed '/ base\(-devel\)\?$/d' "${tmp_dir}/known-groups"
-      } | \
-        sort -k2,2 | \
-        join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/new" | \
-        sponge -a "${tmp_dir}/build-order"
+#  output="${webserver_directory}/graphs/${target_package}.png"
+  output="${webserver_directory}/test.png"
 
-      rm "${tmp_dir}/new"
-
-      sort -u "${tmp_dir}/build-order" | \
-        sponge "${tmp_dir}/build-order"
-
-      current_sum=$(sha512sum "${tmp_dir}/build-order")
-
-    done
-
-  fi
+  # shellcheck disable=SC2016
+  {
+    printf 'CREATE TEMPORARY TABLE `relevant_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+    printf 'CREATE TEMPORARY TABLE `relevant_binary_packages_copy` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+    printf 'CREATE TEMPORARY TABLE `relevant_install_targets` (`id` BIGINT, UNIQUE KEY (`id`));\n'
 
-#  grep -v '^\(\S\+\) \1$' "${tmp_dir}/build-order" | \
-#    sponge "${tmp_dir}/build-order"
+    if [ "${target_package}" = 'ALL' ]; then
 
-  # shellcheck disable=SC2129
-  tr ' ' '\n' < \
-    "${tmp_dir}/build-order" | \
-    sort -u | \
-    join -j 1 - "${tmp_dir}/build-list.sorted-by-package" | \
-    while read -r pkg rev mod_rev repo; do
-      if [ -f "${work_dir}/package-states/${pkg}.${rev}.${mod_rev}.${repo}.broken" ]; then
-        printf '1 broken-build-list-package %s\n' "${pkg}"
-      elif [ -f "${work_dir}/package-states/${pkg}.${rev}.${mod_rev}.${repo}.blocked" ]; then
-        printf '2 blocked-build-list-package %s\n' "${pkg}"
-      else
-        printf '3 build-list-package %s\n' "${pkg}"
-      fi
-    done >> \
-    "${tmp_dir}/knots"
+      printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)'
+      printf ' SELECT DISTINCT `binary_packages`.`id`'
+      printf ' FROM `repositories`'
+      mysql_join_repositories_repository_stabilities
+      printf ' AND `repository_stabilities`.`name` IN ("unbuilt","staging")'
+      mysql_join_repositories_binary_packages
+      printf ';\n'
 
-  {
-    sort -u "${work_dir}/deletion-list"
-    tr ' ' '\n' < \
-      "${tmp_dir}/build-order" | \
-      sort -u
-  } | \
-    sort | \
-    uniq -d | \
-    xargs -r printf '5 deletion-list-package %s\n' >> \
-    "${tmp_dir}/knots"
+    else
 
-  {
-    awk '{print $2}' "${tmp_dir}/known-split-packages" | \
-      sort -u
-    tr ' ' '\n' < \
-      "${tmp_dir}/build-order" | \
-      sort -u
-  } | \
-    sort | \
-    uniq -d | \
-    xargs -r printf '6 split-package %s\n' >> \
-    "${tmp_dir}/knots"
+      printf 'CALL calculate_dependencies_of_package_upto_first_built_one(from_base64("%s"));\n' \
+        "$(
+          printf '%s' "${target_package}" | \
+            base64 -w0
+        )"
 
-  {
-    cat "${tmp_dir}/known-packages"
-    tr ' ' '\n' < \
-      "${tmp_dir}/build-order" | \
-      sort -u
-  } | \
-    sort | \
-    uniq -d | \
-    xargs -r printf '7 package %s\n' >> \
-    "${tmp_dir}/knots"
+    fi
 
-  {
-    awk '{print $2}' "${tmp_dir}/known-groups" | \
-      sort -u
-    tr ' ' '\n' < \
-      "${tmp_dir}/build-order" | \
-      sort -u
+    printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)'
+    printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`'
+    printf ';\n'
+    printf 'INSERT IGNORE INTO `relevant_install_targets` (`id`)'
+    printf ' SELECT DISTINCT `install_target_providers`.`install_target`'
+    printf ' FROM `relevant_binary_packages`'
+    mysql_join_binary_packages_install_target_providers 'relevant_binary_packages'
+    mysql_join_install_target_providers_dependencies
+    mysql_join_dependencies_dependency_types
+    printf ' AND `dependency_types`.`relevant_for_building`'
+    printf ' JOIN `relevant_binary_packages_copy` ON `relevant_binary_packages_copy`.`id`=`dependencies`.`dependent`'
+    printf ';\n'
+
+    # we return lines with either:
+    # "knot" $type $identifier $label
+    # or
+    # "edge" $type $from_knot $to_knot
+    printf 'SELECT DISTINCT'
+    printf ' "knot",'
+    printf 'IF(`build_assignments`.`is_%s`,"%s-build-list-pkgbase",' \
+      'blocked' 'broken' \
+      'broken' 'broken'
+    printf '"build-list-pkgbase"'
+    printf ')),'
+    printf 'CONCAT("pkgbase:",`package_sources`.`id`),'
+    printf '`package_sources`.`pkgbase`'
+    printf ' FROM `relevant_binary_packages`'
+    printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' AND `repository_stabilities`.`name`="unbuilt"'
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
+    printf ';\n'
+
+    printf 'SELECT DISTINCT'
+    printf ' "knot",'
+    printf 'CONCAT("pkgname-",`repository_stabilities`.`name`),'
+    printf 'CONCAT("pkgname:",`binary_packages`.`id`),'
+    printf '`binary_packages`.`pkgname`'
+    printf ' FROM `relevant_binary_packages`'
+    printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ';\n'
+
+    printf 'SELECT DISTINCT'
+    printf ' "knot","install-target",'
+    printf 'CONCAT("install-target:",`install_targets`.`id`),'
+    printf '`install_targets`.`name`'
+    printf ' FROM `relevant_install_targets`'
+    printf ' JOIN `install_targets` ON `install_targets`.`id`=`relevant_install_targets`.`id`'
+    printf ';\n'
+
+    printf 'SELECT DISTINCT'
+    printf ' "edge","builds"'
+    printf ',CONCAT("%s:",`%s`.`id`)' \
+      'pkgbase' 'package_sources' \
+      'pkgname' 'binary_packages'
+    printf ' FROM `relevant_binary_packages`'
+    printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' AND `repository_stabilities`.`name`="unbuilt"'
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
+    printf ';\n'
+
+    printf 'SELECT DISTINCT'
+    printf ' "edge","provides"'
+    printf ',CONCAT("%s:",`%s`.`id`)' \
+      'pkgname' 'binary_packages' \
+      'install-target' 'install_targets'
+    printf ' FROM `relevant_binary_packages`'
+    printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_install_target_providers
+    printf ' JOIN `relevant_install_targets` ON `relevant_install_targets`.`id`=`install_target_providers`.`install_target`'
+    mysql_join_install_target_providers_install_targets
+    printf ';\n'
+
+    printf 'SELECT DISTINCT'
+    printf ' "edge",CONCAT("depends:",`dependency_types`.`name`)'
+    printf ',CONCAT("%s:",`%s`.`id`)' \
+      'install-target' 'install_targets' \
+      'pkgname' 'binary_packages'
+    printf ' FROM `relevant_binary_packages`'
+    printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_dependencies
+    printf ' JOIN `relevant_install_targets` ON `relevant_install_targets`.`id`=`dependencies`.`depending_on`'
+    mysql_join_dependencies_dependency_types
+    printf ' AND `dependency_types`.`relevant_for_building`'
+    mysql_join_dependencies_install_targets
+    printf ';\n'
   } | \
-    sort | \
-    uniq -d | \
-    xargs -r printf '8 group %s\n' >> \
-    "${tmp_dir}/knots"
-
-  tr ' ' '\n' < \
-    "${tmp_dir}/build-order" | \
-    sort -u | \
-    xargs -r printf '100 unknown %s\n' >> \
+    ${mysql_command} -N --raw --batch | \
+    sed '
+      y/\t/ /
+      /^knot /{
+        s/^\S\+ //
+        w '"${tmp_dir}"'/knots
+        d
+      }
+      /^edge /{
+        s/^\S\+ //
+        w '"${tmp_dir}"'/edges
+        d
+      }
+    ' >&2
+
+  mkdir "${tmp_dir}/find-identical"
+
+  awk '{
+    print $1 >> "'"${tmp_dir}"'/find-identical/characteristics-"$2
+  }' < \
     "${tmp_dir}/knots"
-
-  sort -k3,3 -k1n,1 "${tmp_dir}/knots" | \
-    uniq -f2 | \
-    awk '{print $3 " " $1 " " $2}' | \
-    sort -k2,3 | \
+  awk '{
+    print $1 " X " $3 >> "'"${tmp_dir}"'/find-identical/characteristics-"$2
+    print $1 " " $2 " X" >> "'"${tmp_dir}"'/find-identical/characteristics-"$3
+  }' < \
+    "${tmp_dir}/edges"
+
+  modifier=$(
+    find "${tmp_dir}/find-identical" -maxdepth 1 \
+      -type f \
+      -name 'characteristics-*' | \
+      while read -r file; do
+        printf '%s ' "${file##*/}"
+        sort -u "${file}" | \
+          sha512sum | \
+          awk '{print $1}'
+      done | \
+      sort -k2,2 | \
+      uniq -Df1 | \
+      sed 's,^characteristics-,,' | \
+      sed '
+        :a
+          $!N
+          s/^\(\S\+\) \(\S\+\)\n\(\S\+ \)\2$/\1+\3\2/
+          ta
+          P
+          D
+      ' | \
+      cut -d' ' -f1 | \
+      sed '
+        s/^/s,\\(/
+        s/+\([^+[:space:]]\+\)$/\\)\\(\\s\\|$\\),\1\\2,g;/
+        s/+/\\|/g
+      '
+  )
+  rm -rf --one-file-system "${tmp_dir}/find-identical"
+
+  sed "${modifier}" "${tmp_dir}/knots" | \
+    sort -k2,2 -k1,1 | \
+    sed '
+      :a
+        $!N
+        s/^\(\S\+ \S\+ \)\(\S\+\)\n\1\(\S\+\)$/\1\2<nl>\3/
+        ta
+        P
+        D
+    ' | \
     sponge "${tmp_dir}/knots"
 
-  mkdir "${tmp_dir}/neighbours"
-  cat "${tmp_dir}/knots" "${tmp_dir}/build-order" | \
-    awk '{
-      print $1 " " $2 > "'"${tmp_dir}/neighbours/"'"$1;
-      print $1 " " $2 > "'"${tmp_dir}/neighbours/"'"$2;
-    }'
-
-  find "${tmp_dir}/neighbours" -maxdepth 1 -type f | \
-    while read -r file; do
-      sed '
-          s@\(^\| \)'"$(str_to_regex "${file##*/}")"'\( \|$\)@\1<knot>\2@
-          s@\(^\| \)'"$(str_to_regex "${file##*/}")"'\( \|$\)@\1<knot>\2@
-        ' "${file}" | \
-        sort -u | \
-        sponge "${file}"
-    done
-
-  find "${tmp_dir}/neighbours" -maxdepth 1 -type f -exec \
-    sha512sum {} \; | \
-    sed 's|^\(\S\+\)  .*/\([^/]\+\)$|\2 \1|' | \
-    sort -k2,2 | \
-    uniq -f1 -D | \
-    awk '{print $2 " " $1}' > \
-    "${tmp_dir}/sums"
-
-  rm -rf --one-file-system "${tmp_dir}/neighbours"
-
-  sed '
-    :a
-    $!N
-    s|^\(\S\+\) \([^\n]\+\)\n\1 |\1 \2<nl>|
-    ta
-    P
-    D
-  ' "${tmp_dir}/sums" | \
-  join -1 1 -2 1 -o 2.2,1.2 - "${tmp_dir}/sums" | \
-  while read -r original replacement; do
-    sed -i '
-      s@\( \|^\)'"$(str_to_regex "${original}")"'\( \|$\)@\1'"${replacement}"'\2@
-      s@\( \|^\)'"$(str_to_regex "${original}")"'\( \|$\)@\1'"${replacement}"'\2@
-    ' "${tmp_dir}/build-order" "${tmp_dir}/knots"
-  done
-
-  grep -v '^\(\S\+\) \1$' "${tmp_dir}/build-order" | \
+  sed "${modifier}" "${tmp_dir}/edges" | \
     sort -u | \
-    sponge "${tmp_dir}/build-order"
-  sort -u "${tmp_dir}/knots" | \
-    sponge "${tmp_dir}/knots"
+    sponge "${tmp_dir}/edges"
 
   {
     printf '%s\n' \
       'digraph dependencies {' \
       '  fontname=dejavu;'
 
-    cut -d' ' -f1,3 < \
-      "${tmp_dir}/knots" | \
-      while read -r who what; do
-        if [ "${what}" = 'broken-build-list-package' ]; then
+    # knots: $type $identifier $label
+    # edges: $type $from_knot $to_knot
+
+    while read -r type id label; do
+      case "${type}" in
+        'broken-build-list-pkgbase')
           color='#ff0000'
-        elif [ "${what}" = 'blocked-build-list-package' ]; then
+        ;;
+        'blocked-build-list-pkgbase')
           color='#d00000'
-        elif [ "${what}" = 'build-list-package' ]; then
+        ;;
+        'build-list-pkgbase')
           color='#800000'
-        elif [ "${what}" = 'deletion-list-package' ]; then
-          color='#808080'
-        elif [ "${what}" = 'group' ]; then
-          color='#0000ff'
-        elif [ "${what}" = 'package' ]; then
-          color='#000000'
-        elif [ "${what}" = 'split-package' ]; then
-          color='#8080ff'
-        elif [ "${what}" = 'staging-package' ]; then
+        ;;
+        'pkgname-unbuilt')
+          color='#800000'
+        ;;
+        'pkgname-staging')
           color='#008000'
-        elif [ "${what}" = 'target-package' ]; then
-          color='#00ff00'
-        else
+        ;;
+        'pkgname-stable'|'pkgname-testing'|'pkgname-standalone')
+          color='#000000'
+        ;;
+        'install-target')
+          color='#000080'
+        ;;
+        *)
           color='#ff80ff'
-        fi
-        printf '  "%s" [fontcolor="%s"];\n' "${who}" "${color}"
-      done
+          label="${label} (${type})"
+        ;;
+      esac
+      printf '  "%s" [label="%s", fontcolor="%s"];\n' "${id}" "${label}" "${color}"
+    done < \
+      "${tmp_dir}/knots"
 
-    sed 's|\\|\\\\|g' "${tmp_dir}/build-order" | \
-      xargs -rn2 printf '  "%s" -> "%s";\n' | \
-      sort -u
+    while read -r type from_knot to_knot; do
+      printf '"%s" -> "%s";\n' \
+        "${from_knot}" "${to_knot}"
+    done < \
+      "${tmp_dir}/edges"
 
     printf '%s\n' \
       '}'
@@ -421,16 +310,12 @@ for target_package in "$@"; do
   ' "${tmp_dir}/input"
 
   line_count=$(wc -l < "${tmp_dir}/input")
-  if [ "${line_count}" -gt 500 ] && [ "${target_package}" != 'ALL' ]; then
-    sed -i '/"base\(-devel\)\?"/d' "${tmp_dir}/input"
-    line_count=$(wc -l < "${tmp_dir}/input")
-    if [ "${line_count}" -gt 700 ]; then
-      rm -f "${output}"
-      >&2 printf 'Skipping graph for "%s" - would be too big (%d).\n' \
-        "${target_package}" \
-        "${line_count}"
-      continue
-    fi
+  if [ "${target_package}" != 'ALL' ] && [ "${line_count}" -gt 1000 ]; then
+    rm -f "${output}"
+    >&2 printf 'Skipping graph for "%s" - would be too big (%d).\n' \
+      "${target_package}" \
+      "${line_count}"
+    continue
   fi
   printf 'small enough (%s): %d\n' \
     "${target_package}" \
-- 
cgit v1.2.3-54-g00ecf


From d3e37311cfd3045a222230d867838116d484a2e9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 15:36:09 +0100
Subject: bin/show-dependencies: output to real file

---
 bin/show-dependencies | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bin/show-dependencies b/bin/show-dependencies
index bce97d1..4e90258 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -56,8 +56,7 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
 for target_package in "$@"; do
 
-#  output="${webserver_directory}/graphs/${target_package}.png"
-  output="${webserver_directory}/test.png"
+  output="${webserver_directory}/graphs/${target_package}.png"
 
   # shellcheck disable=SC2016
   {
-- 
cgit v1.2.3-54-g00ecf


From 3c876e32b322c15326b5835bb4fef1a02adbcbb6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 19:38:03 +0100
Subject: lib/common-functions: save date to command-log; skip logging if
 buildmaster is insane

---
 lib/common-functions | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/common-functions b/lib/common-functions
index 80f252c..26151d1 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -15,14 +15,16 @@ if [ -z "${base_dir}" ]; then
   . 'conf/default.conf'
 fi
 
-{
-  printf 'sourcing common-functions for %s\n' "$0"
-  printf '%s parameters:' "$#"
-  printf ' "%s"' "$@"
-  printf '\n'
-} >> \
-  "${work_dir}/command-log"
-
+if [ ! -s "${work_dir}/build-master-sanity" ]; then
+  {
+    date
+    printf 'sourcing common-functions for %s\n' "$0"
+    printf '%s parameters:' "$#"
+    printf ' "%s"' "$@"
+    printf '\n'
+  } >> \
+    "${work_dir}/command-log"
+fi
 
 # find_pkgbuilds package repository git_repository git_revision mod_git_revision
 # find the PKGBUILD and modification of $package from $repository
-- 
cgit v1.2.3-54-g00ecf


From 6dd8f08841badc851ee88ebcf0364068a4a42e2e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 20:31:30 +0100
Subject: bin/seed-build-list: ignore to-be-deleted packages, too

---
 bin/seed-build-list | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 284c555..26b386d 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -159,10 +159,18 @@ fi
   fi
   # packages on the build-list or deletion-list should be ignored
   printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)'
-  printf ' SELECT DISTINCT `binary_packages`.`pkgname`'
-  printf ' FROM `binary_packages`'
-  mysql_join_binary_packages_repositories
-  printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list");\n'
+  printf ' SELECT DISTINCT `ignore_bin`.`pkgname`'
+  printf ' FROM `binary_packages` AS `ignore_bin`'
+  mysql_join_binary_packages_repositories 'ignore_bin'
+  printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list")'
+  # packages with no not-to-be-deleted and at least on to-be-deleted version should be ignored
+  printf ' OR ('
+  printf '`ignore_bin`.`is_to_be_deleted`'
+  printf ' AND NOT EXISTS ('
+    printf 'SELECT * FROM `binary_packages` AS `other_bin`'
+    printf ' WHERE NOT `other_bin`.`is_to_be_deleted`'
+    printf ' AND `other_bin`.`pkgname`=`ignore_bin`.`pkgname`'
+  printf '));\n'
 } | \
   sponge "${tmp_dir}/ignore-packages"
 
-- 
cgit v1.2.3-54-g00ecf


From c676d3e6e7e1fb1c822fdcea37f790e22aa58200 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 26 Feb 2018 20:43:16 +0100
Subject: bin/seed-build-list: also write to build-list

---
 bin/seed-build-list | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 26b386d..465ef5d 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -309,7 +309,8 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     flock -s 8
 
     while read -r pkgbase git_rev mod_git_rev repo; do
-      printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
+      printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \
+        "${work_dir}/build-list"
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
       generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
-- 
cgit v1.2.3-54-g00ecf


From f22acc6a42126250fb0aa1844e20ec0abc6210b3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 27 Feb 2018 07:13:45 +0100
Subject: bin/get-package-updates: add TODO

---
 bin/get-package-updates | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 70f62e8..1accadf 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -16,6 +16,9 @@
 
 # TODO: keep database clean in case of abort
 
+# TODO: It seems, updating present build assignments in the database is borken
+# ... they seem to get deleted
+
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
-- 
cgit v1.2.3-54-g00ecf


From cbafb5816805fb00f3d309e99e4477d17e0607d4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 27 Feb 2018 09:56:56 +0100
Subject: bin/get-package-updates: fix typo in comment

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1accadf..2fd0993 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -16,7 +16,7 @@
 
 # TODO: keep database clean in case of abort
 
-# TODO: It seems, updating present build assignments in the database is borken
+# TODO: It seems, updating present build assignments in the database is broken
 # ... they seem to get deleted
 
 # shellcheck disable=SC2016
-- 
cgit v1.2.3-54-g00ecf


From 3567c64454903149b78fe35952e91c18c35af264 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 28 Feb 2018 20:41:35 +0100
Subject: bin/bootstrap-mysql: grant create temporary tables to
 webserver@localhost

---
 bin/bootstrap-mysql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index a172d82..b23bba4 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -304,7 +304,8 @@ if [ ! "$1" = 'slim' ]; then
       'SELECT' 'buildmaster.*' \
       'SHOW VIEW' 'buildmaster.*' \
       'UPDATE' 'buildmaster.*'
-    printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \
+    printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \
+      'CREATE TEMPORARY TABLES' 'buildmaster.*' \
       'SELECT' 'buildmaster.*' \
       'SHOW VIEW' 'buildmaster.*'
     printf 'FLUSH PRIVILEGES;\n'
-- 
cgit v1.2.3-54-g00ecf


From 6c68e125ed9307fa6628937e4ee162c641db1297 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 28 Feb 2018 21:56:02 +0100
Subject: web-scripts/dependencies.php new

---
 web-scripts/dependencies.php | 145 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100644 web-scripts/dependencies.php

diff --git a/web-scripts/dependencies.php b/web-scripts/dependencies.php
new file mode 100644
index 0000000..66221bd
--- /dev/null
+++ b/web-scripts/dependencies.php
@@ -0,0 +1,145 @@
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$match = "";
+
+if (isset($_GET["a"]))
+  $match .= " AND `architectures`.`name`=from_base64(\"" . base64_encode($_GET["a"]) . "\")";
+if (isset($_GET["p"]))
+  $match .= " AND `binary_packages`.`pkgname`=from_base64(\"" . base64_encode($_GET["p"]) . "\")";
+if (isset($_GET["r"]))
+  $match .= " AND `repositories`.`name`=from_base64(\"" . base64_encode($_GET["r"]) . "\")";
+
+$ignore_install_targets = " AND NOT `install_targets`.`name` IN (\"base\",\"base-devel\")";
+
+if (! $result = $mysql -> query(
+  "CREATE TEMPORARY TABLE `cons` (" .
+    "`dep` BIGINT, " .
+    "`itp` BIGINT, " .
+    "UNIQUE KEY `content` (`dep`,`itp`)" .
+  ")"))
+  die($mysql->error);
+
+if (! $result = $mysql -> query(
+  "INSERT IGNORE INTO `cons` (`dep`,`itp`)" .
+  " SELECT `dependencies`.`id`,`install_target_providers`.`id`".
+  " FROM `binary_packages`" .
+  " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+  " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+  $match .
+  " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+  " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
+  $ignore_install_targets .
+  " JOIN `install_target_providers` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`"
+  ))
+  die($mysql->error);
+
+if (! $result = $mysql -> query(
+  "INSERT IGNORE INTO `cons` (`dep`,`itp`)" .
+  " SELECT `dependencies`.`id`,`install_target_providers`.`id`".
+  " FROM `binary_packages`" .
+  " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+  " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+  $match .
+  " JOIN `install_target_providers` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
+  " JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`"
+  ))
+  die($mysql->error);
+
+unset($knots);
+unset($edges);
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `install_target_providers`.`install_target`,`install_target_providers`.`package`" .
+  " FROM `cons`" .
+  " JOIN `install_target_providers` ON `cons`.`itp`=`install_target_providers`.`id`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $edges .= "\"p" . $row["package"] . "\" -> \"i" . $row["install_target"] . "\" [color = \"#000080\"];\n";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `dependencies`.`dependent`,`dependencies`.`depending_on`,`dependency_types`.`name`" .
+  " FROM `cons`" .
+  " JOIN `dependencies` ON `cons`.`dep`=`dependencies`.`id`" .
+  " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $edges .= "\"i" . $row["depending_on"] . "\" -> \"p" . $row["dependent"] . "\" [taillabel = \"" . $row["name"] . "\"];\n";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `install_targets`.`id`,`install_targets`.`name`" .
+  " FROM `cons`" .
+  " JOIN `dependencies` ON `cons`.`dep`=`dependencies`.`id`" .
+  " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $knots .= "\"i" . $row["id"] . "\" [label = \"" . $row["name"] . "\", fontcolor=\"#000080\"];\n";
+
+$pkgfile_query =
+  "CONCAT(".
+    "`repositories`.`name`,\"/\"," .
+    "`binary_packages`.`pkgname`,\"-\"," .
+    "IF(`binary_packages`.`epoch`=0,\"\",CONCAT(`binary_packages`.`epoch`,\":\"))," .
+    "`binary_packages`.`pkgver`,\"-\"," .
+    "`binary_packages`.`pkgrel`,\".\"," .
+    "`binary_packages`.`sub_pkgrel`,\"-\"," .
+    "`architectures`.`name`" .
+  ") AS `filename`";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `binary_packages`.`id`," . $pkgfile_query .
+  " FROM `cons`" .
+  " JOIN `dependencies` ON `cons`.`dep`=`dependencies`.`id`" .
+  " JOIN `binary_packages` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+  " JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`" .
+  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\"];\n";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `binary_packages`.`id`," . $pkgfile_query .
+  " FROM `cons`" .
+  " JOIN `install_target_providers` ON `cons`.`itp`=`install_target_providers`.`id`" .
+  " JOIN `binary_packages` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
+  " JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`" .
+  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\"];\n";
+
+$knots = str_replace("\$","\\\$",$knots);
+$edges = str_replace("\$","\\\$",$edges);
+
+header ("Content-type: image/png");
+passthru(
+  "dot -Tpng -o/dev/stdout /dev/stdin <<EOF\n" .
+  "digraph dependencies {\n" .
+  "rankdir=LR;\n" .
+  "fontname=dejavu;\n" .
+  $knots .
+  $edges .
+  "}\n" .
+  "EOF\n"
+);
+
+?>
-- 
cgit v1.2.3-54-g00ecf


From 1b15a4704e80853b99eef83a3e26e7735bec2f54 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 28 Feb 2018 22:32:07 +0100
Subject: web-scripts/dependencies.php: limit to 50 connections each

---
 web-scripts/dependencies.php | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/web-scripts/dependencies.php b/web-scripts/dependencies.php
index 66221bd..4f12c52 100644
--- a/web-scripts/dependencies.php
+++ b/web-scripts/dependencies.php
@@ -34,7 +34,8 @@ if (! $result = $mysql -> query(
   " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
   " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
   $ignore_install_targets .
-  " JOIN `install_target_providers` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`"
+  " JOIN `install_target_providers` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`" .
+  " LIMIT 50"
   ))
   die($mysql->error);
 
@@ -46,7 +47,8 @@ if (! $result = $mysql -> query(
   " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
   $match .
   " JOIN `install_target_providers` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
-  " JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`"
+  " JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`" .
+  " LIMIT 50"
   ))
   die($mysql->error);
 
-- 
cgit v1.2.3-54-g00ecf


From 6047f2686cfe623e08fbaf4eae0a552ded731eb1 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Mar 2018 08:37:27 +0100
Subject: switch from calling ${mysql_command} to calling the wrapper function
 mysql_run_query()

---
 bin/bootstrap-mysql                |  4 ++--
 bin/build-master-status            |  6 +++---
 bin/build-master-status-from-mysql | 16 ++++++++--------
 bin/check-bugtracker               |  4 ++--
 bin/copy-to-build-support          |  4 ++--
 bin/db-update                      |  6 +++---
 bin/get-assignment                 | 10 +++++-----
 bin/get-package-updates            | 12 ++++++------
 bin/modify-package-state           |  4 ++--
 bin/prioritize-build-list          |  2 +-
 bin/return-assignment              | 14 +++++++-------
 bin/seed-build-list                |  6 +++---
 bin/show-dependencies              |  4 ++--
 bin/slave-build-connect            |  2 +-
 bin/why-dont-you                   |  2 +-
 lib/common-functions               |  4 ++--
 lib/mysql-functions                | 34 ++++++++++++++++++++--------------
 17 files changed, 70 insertions(+), 64 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index b23bba4..236b0b7 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -590,7 +590,7 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \
     printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
       "$(printf '%s' "${repository}" | base64 -w0)"
   done | \
-  ${mysql_command}
+  mysql_run_query
 
 grep '^\('"$(
   # shellcheck disable=SC2086
@@ -665,7 +665,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
     printf ';\n'
     printf 'DROP TABLE `stable_packages`;\n'
   } | \
-    ${mysql_command}
+    mysql_run_query
 fi
 
 mysql_repair_binary_packages_without_build_assignment
diff --git a/bin/build-master-status b/bin/build-master-status
index d74ff8a..9be7823 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -557,12 +557,12 @@ if ${web}; then
         "${line}"
     done < \
       "${tmp_dir}/todos" | \
-      ${mysql_command}
+      mysql_run_query
     # insert unfound todos
     # shellcheck disable=SC2016
     {
       printf 'SHOW CREATE TABLE `todos`' | \
-        ${mysql_command} --raw --batch | \
+        mysql_run_query --raw --batch | \
         sed '
           1d
           2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
@@ -611,7 +611,7 @@ if ${web}; then
       printf 'WHERE `todos`.`id`=`todo_links`.`dependent`'
       printf ');\n'
     } | \
-      ${mysql_command}
+      mysql_run_query
     rm -f "${tmp_dir}/todos"
   fi
 
diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index b55ca65..fa54177 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -27,7 +27,7 @@ fi
   mysql_join_binary_packages_repositories
   printf ' WHERE `repositories`.`name`="build-list"'
 } | \
-  ${mysql_command} --batch | \
+  mysql_run_query --batch | \
   sed '
     1d
     y/\t/ /
@@ -48,7 +48,7 @@ sort "${work_dir}/build-list" > \
   mysql_join_build_assignments_package_sources
   mysql_join_package_sources_upstream_repositories
 } | \
-  ${mysql_command} --raw --batch | \
+  mysql_run_query --raw --batch | \
   sed '
     1d
     y/\t/ /
@@ -72,7 +72,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin
   printf ' WHERE `repositories`.`name`="deletion-list"'
   printf ' OR `binary_packages`.`is_to_be_deleted`;\n'
 } | \
-  ${mysql_command} --batch | \
+  mysql_run_query --batch | \
   sed '
     1d
     y/\t/ /
@@ -90,7 +90,7 @@ sort "${work_dir}/deletion-list" > \
   mysql_join_build_assignments_build_dependency_loops
   printf ';\n'
 } | \
-  ${mysql_command} --batch | \
+  mysql_run_query --batch | \
   sed '
     1d
     y/\t/ /
@@ -121,7 +121,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
 } | \
-  ${mysql_command} --batch | \
+  mysql_run_query --batch | \
   sed '
     1d
     s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
@@ -156,7 +156,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
 } | \
-  ${mysql_command} --batch | \
+  mysql_run_query --batch | \
   sed '
     1d
     s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
@@ -190,7 +190,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   mysql_join_package_sources_upstream_repositories
   printf ' WHERE `dependency_types`.`relevant_for_building`'
 } | \
-  ${mysql_command} --raw --batch | \
+  mysql_run_query --raw --batch | \
   sed '
     1d
     y/\t/ /
@@ -210,7 +210,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     mysql_join_package_sources_build_assignments
     mysql_join_build_assignments_binary_packages
   } | \
-    ${mysql_command} --raw --batch | \
+    mysql_run_query --raw --batch | \
     sed '
       1d
       y/\t/ /
diff --git a/bin/check-bugtracker b/bin/check-bugtracker
index d6e92e5..f15757b 100755
--- a/bin/check-bugtracker
+++ b/bin/check-bugtracker
@@ -26,7 +26,7 @@ bug_list=$(
   printf ' FROM `repository_stabilities`'
   printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL'
 } | \
-  ${mysql_command} --raw --batch | \
+  mysql_run_query --raw --batch | \
   sed '1d' | \
   while read -r stability_id category; do
     for has_issues in '1:' '0:NOT '; do
@@ -62,4 +62,4 @@ bug_list=$(
       printf '"");\n'
     done
   done | tee /dev/stderr | \
-  ${mysql_command}
+  mysql_run_query
diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 808d041..0b87107 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -44,7 +44,7 @@ sed -n '
           "$(printf '%s' "${architecture}" | base64 -w0)"
         printf ' LIMIT 1;\n'
       } | \
-        ${mysql_command} --raw --batch | \
+        mysql_run_query --raw --batch | \
         sed '
           1d
           y/\t/ /
@@ -123,7 +123,7 @@ ${master_mirror_rsync_command} \
   "${master_mirror_rsync_directory}/i686/build-support/"
 
 if [ -s "${tmp_dir}/mysql-command" ]; then
-  ${mysql_command} < \
+  mysql_run_query < \
     "${tmp_dir}/mysql-command"
 fi
 
diff --git a/bin/db-update b/bin/db-update
index de263be..534c245 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -370,7 +370,7 @@ move_packages() {
     if ${no_action}; then
       sed 's|^|mysql: |'
     else
-      ${mysql_command}
+      mysql_run_query
     fi
 
   ${dummynator} trigger_mirror_refreshs
@@ -457,7 +457,7 @@ errors=$(
       printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
       printf ' WHERE `repository_stabilities`.`name` = "testing"'
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
         p
@@ -496,7 +496,7 @@ errors=$(
       printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
       printf ' WHERE `repository_stabilities`.`name` = "staging"'
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
         p
diff --git a/bin/get-assignment b/bin/get-assignment
index e3299f6..bc67fab 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -58,7 +58,7 @@ hand_out_assignment() {
       "$(printf '%s' "$4" | base64 -w0)"
     printf ' LIMIT 1;\n'
   } | \
-    ${mysql_command} --raw --batch | \
+    mysql_run_query --raw --batch | \
     sed '
       1d
       y/\t/ /
@@ -104,7 +104,7 @@ hand_out_assignment() {
     printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \
       "$(printf '%s' "$4" | base64 -w0)"
   } | \
-    ${mysql_command}
+    mysql_run_query
 
   # lock every loop this package breaks
   find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
@@ -157,7 +157,7 @@ currently_building=$(
     printf ' AND `repositories`.`name`="build-list"'
     printf ' LIMIT 1;\n'
   } | \
-    ${mysql_command} --batch --raw | \
+    mysql_run_query --batch --raw | \
     sed '1d'
 )
 
@@ -214,7 +214,7 @@ next_building=$(
     printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
   } | \
-    ${mysql_command} --batch --raw | \
+    mysql_run_query --batch --raw | \
     sed '
       1d
       y/\t/ /
@@ -238,7 +238,7 @@ count_pending=$(
     printf ' AND `build_assignments`.`is_blocked` IS NULL'
     printf ';\n'
   } | \
-    ${mysql_command} --batch --raw | \
+    mysql_run_query --batch --raw | \
     sed '1d'
 )
 
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 2fd0993..11f384c 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -129,7 +129,7 @@ delete_package() {
       printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \
         "$(printf '%s' "$1" | base64 -w0)"
     } | \
-      ${mysql_command}
+      mysql_run_query
   fi
 }
 
@@ -144,7 +144,7 @@ clean_up() {
       printf ' WHERE `repositories`.`name`="to-be-decided"'
     printf ');\n'
   } | \
-    ${mysql_command}
+    mysql_run_query
 }
 tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)
 trap 'clean_up' EXIT
@@ -353,7 +353,7 @@ done | \
               base64 -w0
           )"
       } | \
-        ${mysql_command}
+        mysql_run_query
       mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       continue
@@ -391,7 +391,7 @@ black_listed_new=$(
       printf ' FROM `package_sources`'
       printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '1d'
   } | \
     sort -u
@@ -443,7 +443,7 @@ while [ -n "${black_listed_new}" ]; do
           printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
         printf ');\n'
       } | \
-        ${mysql_command} --raw --batch | \
+        mysql_run_query --raw --batch | \
         sed '1d' | \
         sort -u
     } | \
@@ -604,7 +604,7 @@ fi
   printf ')'
   printf ' WHERE `from_repo`.`name`="to-be-decided";\n'
 } | \
-  ${mysql_command}
+  mysql_run_query
 
 # update loop list in database (beware, the packages are expected to be in "build-list",
 # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list".
diff --git a/bin/modify-package-state b/bin/modify-package-state
index b0ba53e..8c5fac7 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -225,7 +225,7 @@ while read -r package reason; do
     ;;
   esac
   if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \
-    ${mysql_command} --raw --batch | \
+    mysql_run_query --raw --batch | \
     sed '1d' | \
     grep -qxF '1'; then
     # shellcheck disable=SC2016
@@ -242,7 +242,7 @@ while read -r package reason; do
       printf ' AND NOT `b`.`is_tested`'
       printf ' AND NOT `repositories`.`name`="deletion-list";\n'
     } | \
-      ${mysql_command}
+      mysql_run_query
     case "${action}" in
       'block')
         printf '%s' "${reason}" > \
diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list
index e8dcf0a..1ff3d5a 100755
--- a/bin/prioritize-build-list
+++ b/bin/prioritize-build-list
@@ -46,7 +46,7 @@ fi
     "$(printf '%s' "$1" | base64 -w0)"
   printf ' AND `repositories`.`name`="build-list";\n'
 } | \
-  ${mysql_command}
+  mysql_run_query
 
 # Remove the lock file
 
diff --git a/bin/return-assignment b/bin/return-assignment
index db433ee..ed29f34 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then
         "$(printf '%s' "$4" | base64 -w0)"
       printf ' AND `repositories`.`name`="build-list"'
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
         y/\t/ /
@@ -111,7 +111,7 @@ if [ "$5" = 'ERROR' ]; then
           printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")'
           printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`'
         } | \
-          ${mysql_command} --raw --batch | \
+          mysql_run_query --raw --batch | \
           sed '1d'
       )
       for saved_build_log in ${saved_build_logs}; do
@@ -164,7 +164,7 @@ if [ "$5" = 'ERROR' ]; then
           base64 -w0
       )"
   } | \
-    ${mysql_command}
+    mysql_run_query
 
   # shellcheck disable=SC2154
   sed -i '
@@ -283,7 +283,7 @@ if ! {
     printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \
       "$(printf '%s' "$5" | base64 -w0)"
   } | \
-    ${mysql_command} --raw --batch | \
+    mysql_run_query --raw --batch | \
     sed '1d' | \
     grep -vxF '0'; then
   >&2 echo 'Sorry, the sent package is outdated.'
@@ -441,7 +441,7 @@ package_errors=$(
       printf ' AND `repositories`.`name`="build-list"'
       printf ';\n'
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
       '
@@ -513,7 +513,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_
     printf ';\n'
   done
 } | \
-  ${mysql_command}
+  mysql_run_query
 
 # move packages
 destination=$(official_or_community "$1.$2.$3.$4" 'staging')
@@ -605,7 +605,7 @@ fi
   printf ');\n'
   printf 'DROP TABLE `loops_to_delete`;\n'
 } | \
-  ${mysql_command}
+  mysql_run_query
 rm -f \
   "${work_dir}/package-states/$1.$2.$3.$4.locked" \
   "${work_dir}/package-states/$1.$2.$3.$4.broken"
diff --git a/bin/seed-build-list b/bin/seed-build-list
index 465ef5d..b9ae193 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then
       printf ' FROM `binary_packages`'
       mysql_join_binary_packages_architectures
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
         s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/
@@ -206,7 +206,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`'
   printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n'
 } | \
-  ${mysql_command} --raw --batch | \
+  mysql_run_query --raw --batch | \
   sed '1d' | \
   while read -r pkgname; do
     content=$(
@@ -294,7 +294,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`'
   printf ');\n'
 } | \
-  ${mysql_command} --raw --batch | \
+  mysql_run_query --raw --batch | \
   sort -u | \
   sed '
     y/\t/ /
diff --git a/bin/show-dependencies b/bin/show-dependencies
index 4e90258..9e94fe3 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -10,7 +10,7 @@
 if [ $# -eq 0 ]; then
   broken=$(
     printf 'CALL show_broken_packages_and_dependencies;\n' | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
         s/\s.*$//
@@ -178,7 +178,7 @@ for target_package in "$@"; do
     mysql_join_dependencies_install_targets
     printf ';\n'
   } | \
-    ${mysql_command} -N --raw --batch | \
+    mysql_run_query -N --raw --batch | \
     sed '
       y/\t/ /
       /^knot /{
diff --git a/bin/slave-build-connect b/bin/slave-build-connect
index 119592e..f8ee9ad 100755
--- a/bin/slave-build-connect
+++ b/bin/slave-build-connect
@@ -29,7 +29,7 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \
           base64 -w0
       )"
   } | \
-    ${mysql_command}
+    mysql_run_query
 
   slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}"
 
diff --git a/bin/why-dont-you b/bin/why-dont-you
index d95f2a7..b5d73c5 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -85,7 +85,7 @@ case "${action}" in
       printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`'
       printf ';\n'
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
         y/\t/ /
diff --git a/lib/common-functions b/lib/common-functions
index 26151d1..740c266 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -501,7 +501,7 @@ remove_old_package_versions() {
       "$(printf '%s' "${repository}" | base64 -w0)"
     printf ';\n'
   } | \
-    ${mysql_command} --raw --batch | \
+    mysql_run_query --raw --batch | \
     sed '
       /^\S\+\sCONCAT(/d
       y/\t/ /
@@ -680,7 +680,7 @@ remove_old_package_versions() {
           sed 's/ AND$//'
         printf ';\n'
       done | \
-      ${mysql_command}
+      mysql_run_query
 
     sed '
       s| \(\S\+\)$|-\1|
diff --git a/lib/mysql-functions b/lib/mysql-functions
index 185d9a9..6cbc502 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -22,6 +22,12 @@ base64_encode_each() {
   done
 }
 
+# mysql_run_query
+# wrapper function to query mysql
+mysql_run_query() {
+  ${mysql_command} "$@"
+}
+
 # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
 
 # shellcheck disable=SC2086
@@ -69,7 +75,7 @@ mysql_add_package_source() {
     printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \
       "${values##* }"
   } | \
-    ${mysql_command}
+    mysql_run_query
 }
 
 # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository
@@ -113,7 +119,7 @@ mysql_add_binary_package() {
       s|, )|)|g
       s|, FROM| FROM|g
     ' | \
-    ${mysql_command}
+    mysql_run_query
 }
 
 # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel
@@ -162,7 +168,7 @@ mysql_show_binary_package() {
       s|, FROM| FROM|g
       s|AND;|;|g
     ' | \
-    ${mysql_command} --html
+    mysql_run_query --html
 }
 
 # mysql_add_install_target $install_target
@@ -180,7 +186,7 @@ mysql_add_install_target() {
     printf ' VALUES (from_base64("%s"))' \
       "${install_target}"
   } | \
-    ${mysql_command}
+    mysql_run_query
 }
 
 # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository
@@ -626,7 +632,7 @@ mysql_generate_package_metadata() {
         cat "${temp_dir}/add-install-targets-command"
       fi
     } | \
-      ${mysql_command}
+      mysql_run_query
     printf '.' >&2
 
   )
@@ -662,7 +668,7 @@ mysql_sanity_check() {
       'a' 'b'
     printf ';\n'
   } | \
-    ${mysql_command} --raw --batch | \
+    mysql_run_query --raw --batch | \
     sed '
       /^CONCAT("/d
       s,^,<font color="#FF0000">,
@@ -693,7 +699,7 @@ mysql_sanity_check() {
       mysql_join_binary_packages_repositories
       printf ' WHERE `repositories`.`is_on_master_mirror`'
     } | \
-      ${mysql_command} --batch | \
+      mysql_run_query --batch | \
       sed '
         1d
         s,\t,/,
@@ -724,7 +730,7 @@ mysql_find_build_assignment_loops() {
       mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent'
       printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
     } | \
-      ${mysql_command} --raw --batch | \
+      mysql_run_query --raw --batch | \
       sed '
         1d
         y/\t/ /
@@ -754,7 +760,7 @@ mysql_find_build_assignment_loops() {
         "${new_loops}"
     fi
   } | \
-    ${mysql_command}
+    mysql_run_query
 }
 
 # mysql_cleanup [dry]
@@ -844,7 +850,7 @@ mysql_cleanup() {
     printf ' SET `currently_building`=NULL'
     printf ' WHERE `repositories`.`name`!="build-list";\n'
   } | \
-    ${mysql_command}
+    mysql_run_query
 }
 
 # mysql_query_has_pending_dependencies `build_assignment`.`id`
@@ -904,7 +910,7 @@ mysql_repair_binary_packages_without_build_assignment() {
     mysql_join_binary_packages_architectures
     printf ' WHERE `binary_packages`.`build_assignment`<0'
   } | \
-    ${mysql_command} --raw --batch | \
+    mysql_run_query --raw --batch | \
     sed '1d' | \
     while read -r id pkgname arch; do
       pkgname=$(
@@ -933,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() {
           printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
             "$(printf '%s' "${pkgname}" | base64 -w0)"
         } | \
-          ${mysql_command} --raw --batch | \
+          mysql_run_query --raw --batch | \
           sed '1d' | \
           grep -qvxF '0'; then
         pkgbase="${pkgname}"
@@ -960,7 +966,7 @@ mysql_repair_binary_packages_without_build_assignment() {
       printf ' WHERE `binary_packages`.`id`=%s' "${id}"
       printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}"
     done | \
-    ${mysql_command}
+    mysql_run_query
 }
 
 # mysql_remove_duplicate_binary_packages
@@ -1001,7 +1007,7 @@ mysql_remove_duplicate_build_order() {
     printf ' SET `binary_packages`.`repository`=`ren`.`repo`,'
     printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n'
   } | \
-    ${mysql_command}
+    mysql_run_query
 }
 
 # mysql_package_name_query
-- 
cgit v1.2.3-54-g00ecf


From 496c571cd233a8646351ba241c3119ceaf2b8063 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Mar 2018 09:05:51 +0100
Subject: use "mysql -N" instead of cumbersome sedding

---
 bin/build-master-status            |  5 ++---
 bin/build-master-status-from-mysql | 46 ++++++++++----------------------------
 bin/check-bugtracker               |  5 ++---
 bin/copy-to-build-support          |  7 ++----
 bin/db-update                      | 14 ++++--------
 bin/get-assignment                 | 16 +++++--------
 bin/get-package-updates            |  6 ++---
 bin/modify-package-state           |  3 +--
 bin/return-assignment              | 20 +++++------------
 bin/seed-build-list                | 13 ++++-------
 bin/show-dependencies              |  3 +--
 bin/why-dont-you                   |  7 ++----
 lib/common-functions               |  7 ++----
 lib/mysql-functions                | 19 +++++-----------
 14 files changed, 51 insertions(+), 120 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 9be7823..deb58da 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -562,10 +562,9 @@ if ${web}; then
     # shellcheck disable=SC2016
     {
       printf 'SHOW CREATE TABLE `todos`' | \
-        mysql_run_query --raw --batch | \
+        mysql_run_query -N --raw --batch | \
         sed '
-          1d
-          2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
+          1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
         '
       printf ';\n'
       printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES '
diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index fa54177..d1985e7 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -27,11 +27,8 @@ fi
   mysql_join_binary_packages_repositories
   printf ' WHERE `repositories`.`name`="build-list"'
 } | \
-  mysql_run_query --batch | \
-  sed '
-    1d
-    y/\t/ /
-  ' | \
+  mysql_run_query -N --raw --batch | \
+  tr '\t' ' ' | \
   sort > \
   "${tmp_dir}/build-list.mysql"
 sort "${work_dir}/build-list" > \
@@ -48,11 +45,8 @@ sort "${work_dir}/build-list" > \
   mysql_join_build_assignments_package_sources
   mysql_join_package_sources_upstream_repositories
 } | \
-  mysql_run_query --raw --batch | \
-  sed '
-    1d
-    y/\t/ /
-  ' | \
+  mysql_run_query -N --raw --batch | \
+  tr '\t' ' ' | \
   sort > \
   "${tmp_dir}/locked.mysql"
 find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \
@@ -72,11 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin
   printf ' WHERE `repositories`.`name`="deletion-list"'
   printf ' OR `binary_packages`.`is_to_be_deleted`;\n'
 } | \
-  mysql_run_query --batch | \
-  sed '
-    1d
-    y/\t/ /
-  ' | \
+  mysql_run_query -N --raw --batch | \
   sort > \
   "${tmp_dir}/deletion-list.mysql"
 sort "${work_dir}/deletion-list" > \
@@ -90,11 +80,7 @@ sort "${work_dir}/deletion-list" > \
   mysql_join_build_assignments_build_dependency_loops
   printf ';\n'
 } | \
-  mysql_run_query --batch | \
-  sed '
-    1d
-    y/\t/ /
-  ' | \
+  mysql_run_query -N --raw --batch | \
   sort > \
   "${tmp_dir}/loops.mysql"
 find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
@@ -121,9 +107,8 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
 } | \
-  mysql_run_query --batch | \
+  mysql_run_query -N --raw --batch | \
   sed '
-    1d
     s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
   ' | \
   sort -u > \
@@ -156,9 +141,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
 } | \
-  mysql_run_query --batch | \
+  mysql_run_query -N --raw --batch | \
   sed '
-    1d
     s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
   ' | \
   sort -u > \
@@ -190,11 +174,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   mysql_join_package_sources_upstream_repositories
   printf ' WHERE `dependency_types`.`relevant_for_building`'
 } | \
-  mysql_run_query --raw --batch | \
-  sed '
-    1d
-    y/\t/ /
-  ' | \
+  mysql_run_query -N --raw --batch | \
+  tr '\t' ' ' | \
   sort -u > \
   "${tmp_dir}/build-dependencies.mysql"
 {
@@ -210,11 +191,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     mysql_join_package_sources_build_assignments
     mysql_join_build_assignments_binary_packages
   } | \
-    mysql_run_query --raw --batch | \
-    sed '
-      1d
-      y/\t/ /
-    ' | \
+    mysql_run_query -N --raw --batch | \
+    tr '\t' ' ' | \
     while read -r pkgbase git_revision mod_git_revision repository; do
       if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then
         # TODO: consider versions dependencies, too
diff --git a/bin/check-bugtracker b/bin/check-bugtracker
index f15757b..156f438 100755
--- a/bin/check-bugtracker
+++ b/bin/check-bugtracker
@@ -26,8 +26,7 @@ bug_list=$(
   printf ' FROM `repository_stabilities`'
   printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL'
 } | \
-  mysql_run_query --raw --batch | \
-  sed '1d' | \
+  mysql_run_query -N --raw --batch | \
   while read -r stability_id category; do
     for has_issues in '1:' '0:NOT '; do
       printf 'UPDATE `binary_packages`'
@@ -61,5 +60,5 @@ bug_list=$(
         tr '\n' ','
       printf '"");\n'
     done
-  done | tee /dev/stderr | \
+  done | \
   mysql_run_query
diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 0b87107..e447596 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -44,11 +44,8 @@ sed -n '
           "$(printf '%s' "${architecture}" | base64 -w0)"
         printf ' LIMIT 1;\n'
       } | \
-        mysql_run_query --raw --batch | \
-        sed '
-          1d
-          y/\t/ /
-        '
+        mysql_run_query -N --raw --batch | \
+        tr '\t' ' '
     )
     if [ -z "${id}" ]; then
       continue
diff --git a/bin/db-update b/bin/db-update
index 534c245..78ff622 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -457,11 +457,8 @@ errors=$(
       printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
       printf ' WHERE `repository_stabilities`.`name` = "testing"'
     } | \
-      mysql_run_query --raw --batch | \
-      sed '
-        1d
-        p
-      '
+      mysql_run_query -N --raw --batch | \
+      sed 'p'
     # shellcheck disable=SC2086
     printf '%s\n' ${packages_to_force_stabilize} | \
       sort -u
@@ -496,11 +493,8 @@ errors=$(
       printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
       printf ' WHERE `repository_stabilities`.`name` = "staging"'
     } | \
-      mysql_run_query --raw --batch | \
-      sed '
-        1d
-        p
-      '
+      mysql_run_query -N --raw --batch | \
+      sed 'p'
     # shellcheck disable=SC2086
     printf '%s\n' ${packages_to_force_unstage} | \
       sort -u
diff --git a/bin/get-assignment b/bin/get-assignment
index bc67fab..964c6b7 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -58,11 +58,8 @@ hand_out_assignment() {
       "$(printf '%s' "$4" | base64 -w0)"
     printf ' LIMIT 1;\n'
   } | \
-    mysql_run_query --raw --batch | \
-    sed '
-      1d
-      y/\t/ /
-    '
+    mysql_run_query -N --raw --batch | \
+    tr '\t' ' '
 
   {
     # shellcheck disable=SC2154
@@ -157,8 +154,7 @@ currently_building=$(
     printf ' AND `repositories`.`name`="build-list"'
     printf ' LIMIT 1;\n'
   } | \
-    mysql_run_query --batch --raw | \
-    sed '1d'
+    mysql_run_query -N --batch --raw
 )
 
 if [ -n "${currently_building}" ]; then
@@ -214,9 +210,8 @@ next_building=$(
     printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
   } | \
-    mysql_run_query --batch --raw | \
+    mysql_run_query -N --batch --raw | \
     sed '
-      1d
       y/\t/ /
       s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/
     '
@@ -238,8 +233,7 @@ count_pending=$(
     printf ' AND `build_assignments`.`is_blocked` IS NULL'
     printf ';\n'
   } | \
-    mysql_run_query --batch --raw | \
-    sed '1d'
+    mysql_run_query -N --batch --raw
 )
 
 if [ "${count_pending}" -eq 0 ]; then
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 11f384c..e855b19 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -391,8 +391,7 @@ black_listed_new=$(
       printf ' FROM `package_sources`'
       printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
     } | \
-      mysql_run_query --raw --batch | \
-      sed '1d'
+      mysql_run_query -N --raw --batch
   } | \
     sort -u
 )
@@ -443,8 +442,7 @@ while [ -n "${black_listed_new}" ]; do
           printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
         printf ');\n'
       } | \
-        mysql_run_query --raw --batch | \
-        sed '1d' | \
+        mysql_run_query -N --raw --batch | \
         sort -u
     } | \
       grep -vxF '' | \
diff --git a/bin/modify-package-state b/bin/modify-package-state
index 8c5fac7..68b2384 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -225,8 +225,7 @@ while read -r package reason; do
     ;;
   esac
   if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \
-    mysql_run_query --raw --batch | \
-    sed '1d' | \
+    mysql_run_query -N --raw --batch | \
     grep -qxF '1'; then
     # shellcheck disable=SC2016
     {
diff --git a/bin/return-assignment b/bin/return-assignment
index ed29f34..a7e6623 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -70,11 +70,8 @@ if [ "$5" = 'ERROR' ]; then
         "$(printf '%s' "$4" | base64 -w0)"
       printf ' AND `repositories`.`name`="build-list"'
     } | \
-      mysql_run_query --raw --batch | \
-      sed '
-        1d
-        y/\t/ /
-      '
+      mysql_run_query -N --raw --batch | \
+      tr '\t' ' '
   )
   if [ -z "${infos}" ]; then
     >&2 echo 'You do not build this package (anymore) - move on.'
@@ -111,8 +108,7 @@ if [ "$5" = 'ERROR' ]; then
           printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")'
           printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`'
         } | \
-          mysql_run_query --raw --batch | \
-          sed '1d'
+          mysql_run_query -N --raw --batch
       )
       for saved_build_log in ${saved_build_logs}; do
         printf '%s' "${fail_reason_identifiers}" | \
@@ -283,9 +279,8 @@ if ! {
     printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \
       "$(printf '%s' "$5" | base64 -w0)"
   } | \
-    mysql_run_query --raw --batch | \
-    sed '1d' | \
-    grep -vxF '0'; then
+    mysql_run_query -N --raw --batch | \
+    grep -qvxF '0'; then
   >&2 echo 'Sorry, the sent package is outdated.'
   exit 2
 fi
@@ -441,10 +436,7 @@ package_errors=$(
       printf ' AND `repositories`.`name`="build-list"'
       printf ';\n'
     } | \
-      mysql_run_query --raw --batch | \
-      sed '
-        1d
-      '
+      mysql_run_query -N --raw --batch
   } | \
     sort -k2 | \
     uniq -u -f1
diff --git a/bin/seed-build-list b/bin/seed-build-list
index b9ae193..def9d56 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -125,9 +125,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then
       printf ' FROM `binary_packages`'
       mysql_join_binary_packages_architectures
     } | \
-      mysql_run_query --raw --batch | \
+      mysql_run_query -N --raw --batch | \
       sed '
-        1d
         s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/
       '
   } | \
@@ -206,8 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`'
   printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n'
 } | \
-  mysql_run_query --raw --batch | \
-  sed '1d' | \
+  mysql_run_query -N --raw --batch | \
   while read -r pkgname; do
     content=$(
       curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
@@ -294,12 +292,9 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`'
   printf ');\n'
 } | \
-  mysql_run_query --raw --batch | \
+  mysql_run_query -N --raw --batch | \
   sort -u | \
-  sed '
-    y/\t/ /
-    / [0-9a-f]\{40\} [0-9a-f]\{40\} /!d
-  ' | \
+  tr '\t' ' ' | \
   if ${update}; then
     # always block if locked
     exec 9> "${build_list_lock_file}"
diff --git a/bin/show-dependencies b/bin/show-dependencies
index 9e94fe3..ed7ca87 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -10,9 +10,8 @@
 if [ $# -eq 0 ]; then
   broken=$(
     printf 'CALL show_broken_packages_and_dependencies;\n' | \
-      mysql_run_query --raw --batch | \
+      mysql_run_query -N --raw --batch | \
       sed '
-        1d
         s/\s.*$//
       ' | \
       sort -u
diff --git a/bin/why-dont-you b/bin/why-dont-you
index b5d73c5..35b9419 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -85,11 +85,8 @@ case "${action}" in
       printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`'
       printf ';\n'
     } | \
-      mysql_run_query --raw --batch | \
-      sed '
-        1d
-        y/\t/ /
-      ' | \
+      mysql_run_query -N --raw --batch | \
+      tr '\t' ' ' | \
       sort -k7,7 -k6,6 -k5,5 | \
       sed '
         / NULL \S\+$/ b multi-dep
diff --git a/lib/common-functions b/lib/common-functions
index 740c266..ba434ad 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -501,11 +501,8 @@ remove_old_package_versions() {
       "$(printf '%s' "${repository}" | base64 -w0)"
     printf ';\n'
   } | \
-    mysql_run_query --raw --batch | \
-    sed '
-      /^\S\+\sCONCAT(/d
-      y/\t/ /
-    ' | \
+    mysql_run_query -N --raw --batch | \
+    tr '\t' ' ' | \
     expand_version 4 | \
     sort -k4V,4 -k3r,3 | \
     shrink_version 4 | \
diff --git a/lib/mysql-functions b/lib/mysql-functions
index 6cbc502..3e7e19f 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -668,9 +668,8 @@ mysql_sanity_check() {
       'a' 'b'
     printf ';\n'
   } | \
-    mysql_run_query --raw --batch | \
+    mysql_run_query -N --raw --batch | \
     sed '
-      /^CONCAT("/d
       s,^,<font color="#FF0000">,
       s,$,</font>,
     '
@@ -699,9 +698,8 @@ mysql_sanity_check() {
       mysql_join_binary_packages_repositories
       printf ' WHERE `repositories`.`is_on_master_mirror`'
     } | \
-      mysql_run_query --batch | \
+      mysql_run_query -N --raw --batch | \
       sed '
-        1d
         s,\t,/,
         s,\t,-,
         s,\t,:,
@@ -730,11 +728,8 @@ mysql_find_build_assignment_loops() {
       mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent'
       printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
     } | \
-      mysql_run_query --raw --batch | \
-      sed '
-        1d
-        y/\t/ /
-      ' | \
+      mysql_run_query -N --raw --batch | \
+      tr '\t' ' ' | \
       tsort 2>&1 >/dev/null | \
       sed 's/^tsort:\s*//' | \
       {
@@ -910,8 +905,7 @@ mysql_repair_binary_packages_without_build_assignment() {
     mysql_join_binary_packages_architectures
     printf ' WHERE `binary_packages`.`build_assignment`<0'
   } | \
-    mysql_run_query --raw --batch | \
-    sed '1d' | \
+    mysql_run_query -N --raw --batch | \
     while read -r id pkgname arch; do
       pkgname=$(
         printf '%s' "${pkgname}" | \
@@ -939,8 +933,7 @@ mysql_repair_binary_packages_without_build_assignment() {
           printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
             "$(printf '%s' "${pkgname}" | base64 -w0)"
         } | \
-          mysql_run_query --raw --batch | \
-          sed '1d' | \
+          mysql_run_query -N --raw --batch | \
           grep -qvxF '0'; then
         pkgbase="${pkgname}"
       fi
-- 
cgit v1.2.3-54-g00ecf


From 7726164a7bc661bb4c0e5bfa6b81788b375a973b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Mar 2018 09:30:27 +0100
Subject: lib/mysql-functions: mysql_run_query logs everything, now

---
 lib/mysql-functions | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index 3e7e19f..d8ffeb5 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -25,7 +25,28 @@ base64_encode_each() {
 # mysql_run_query
 # wrapper function to query mysql
 mysql_run_query() {
-  ${mysql_command} "$@"
+  {
+    printf '%s %s: ' "$0" "${mysql_command} $*"
+    date
+  } \
+    | tee -a "${work_dir}/mysql.stdin" \
+    | tee -a "${work_dir}/mysql.stdout" \
+    >> "${work_dir}/mysql.stderr"
+  tee -a "${work_dir}/mysql.stdin" \
+    | ${mysql_command} "$@" \
+    2>> "${work_dir}/mysql.stderr" \
+    | tee -a "${work_dir}/mysql.stdout"
+  for s in \
+    "${work_dir}/mysql.stdin" \
+    "${work_dir}/mysql.stdout" \
+    "${work_dir}/mysql.stderr"; do
+    {
+      tail -n 10000 "$s"
+      printf '%s %s done: ' "$0" "${mysql_command} $*"
+      date
+    } \
+      | sponge "$s"
+  done
 }
 
 # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
-- 
cgit v1.2.3-54-g00ecf


From 48c7a60e79436c320627740c1cf5a7bc85a8fe96 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Mar 2018 14:04:31 +0100
Subject: bin/bootstrap-mysql: repository_moves new

---
 bin/bootstrap-mysql | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 236b0b7..c828809 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -136,7 +136,12 @@ if [ ! "$1" = 'slim' ]; then
       ' depending_on MEDIUMINT' \
       'repository_stability_relations MEDIUMINT' \
       ' more_stable MEDIUMINT :repository_stabilities' \
-      ' less_stable MEDIUMINT :repository_stabilities'
+      ' less_stable MEDIUMINT :repository_stabilities' \
+      'repository_moves MEDIUMINT' \
+      ' from_repository MEDIUMINT :repositories' \
+      ' to_repository MEDIUMINT :repositories' \
+      ' upstream_package_repository SMALLINT :upstream_repositories' \
+      ' UNIQUE source from_repository upstream_package_repository'
   )
 
   # shellcheck disable=SC2016
@@ -488,6 +493,29 @@ fi
   } | \
     sed 's| OR ;|;|'
 
+  {
+    printf 'INSERT IGNORE INTO `repository_moves` (`from_repository`,`to_repository`,`upstream_package_repository`)'
+    printf ' SELECT `f`.`id`,`t`.`id`,`u`.`id`'
+    printf ' FROM'
+    printf ' `repositories` AS `%s` JOIN' \
+      'f' 't'
+    printf ' `upstream_repositories` AS `u`'
+    printf ' WHERE '
+    printf '(`f`.`name`="%s" AND `t`.`name`="%s" AND `u`.`name`="%s") OR ' \
+      'staging' 'testing' 'core' \
+      'staging' 'testing' 'extra' \
+      'staging' 'testing' 'multilib' \
+      'community-staging' 'community-testing' 'community' \
+      'community-staging' 'community-testing' 'multilib' \
+      'testing' 'core' 'core' \
+      'testing' 'extra' 'extra' \
+      'testing' 'extra' 'multilib' \
+      'community-testing' 'community' 'community' \
+      'community-testing' 'community' 'multilib'
+    printf ';\n'
+  } | \
+    sed 's| OR ;|;|'
+
   for repo in \
     'core:stable:AQ==' \
     'extra:stable:AQ==' \
-- 
cgit v1.2.3-54-g00ecf


From 9ef68780beab5634b7b322bcb731ba5c9aec0350 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Mar 2018 19:03:27 +0100
Subject: lib/mysql-functions: mysql_run_query: only log if sane

---
 lib/mysql-functions | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index d8ffeb5..a84d74c 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -25,28 +25,32 @@ base64_encode_each() {
 # mysql_run_query
 # wrapper function to query mysql
 mysql_run_query() {
-  {
-    printf '%s %s: ' "$0" "${mysql_command} $*"
-    date
-  } \
-    | tee -a "${work_dir}/mysql.stdin" \
-    | tee -a "${work_dir}/mysql.stdout" \
-    >> "${work_dir}/mysql.stderr"
-  tee -a "${work_dir}/mysql.stdin" \
-    | ${mysql_command} "$@" \
-    2>> "${work_dir}/mysql.stderr" \
-    | tee -a "${work_dir}/mysql.stdout"
-  for s in \
-    "${work_dir}/mysql.stdin" \
-    "${work_dir}/mysql.stdout" \
-    "${work_dir}/mysql.stderr"; do
+  if [ -s "${work_dir}/build-master-sanity" ]; then
+    ${mysql_command} "$@"
+  else
     {
-      tail -n 10000 "$s"
-      printf '%s %s done: ' "$0" "${mysql_command} $*"
+      printf '%s %s: ' "$0" "${mysql_command} $*"
       date
     } \
-      | sponge "$s"
-  done
+      | tee -a "${work_dir}/mysql.stdin" \
+      | tee -a "${work_dir}/mysql.stdout" \
+      >> "${work_dir}/mysql.stderr"
+    tee -a "${work_dir}/mysql.stdin" \
+      | ${mysql_command} "$@" \
+      2>> "${work_dir}/mysql.stderr" \
+      | tee -a "${work_dir}/mysql.stdout"
+    for s in \
+      "${work_dir}/mysql.stdin" \
+      "${work_dir}/mysql.stdout" \
+      "${work_dir}/mysql.stderr"; do
+      {
+        tail -n 10000 "$s"
+        printf '%s %s done: ' "$0" "${mysql_command} $*"
+        date
+      } \
+        | sponge "$s"
+    done
+  fi
 }
 
 # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
-- 
cgit v1.2.3-54-g00ecf


From b2406eb70084fe46fcc76e57818b17c5700c9709 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 4 Mar 2018 21:16:39 +0100
Subject: bin/build-master-status-from-mysql: mark build master as insane if
 check failed

---
 bin/build-master-status-from-mysql | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index d1985e7..7cf1b95 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -245,3 +245,11 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     $ a </body></html>
   ' | \
   sponge "${webserver_directory}/mysql-sanity.html"
+
+if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
+  [ ! -s "${work_dir}/build-master-sanity" ]; then
+  printf 'deep42thought: my database is dirty again ...\n' | \
+    sponge "${irc_dir}/#archlinux-ports/in"
+  echo 'build master is insane' > \
+    "${work_dir}/build-master-sanity"
+fi
-- 
cgit v1.2.3-54-g00ecf


From dff5bf22329ba030c70600fb0fa59087990f09a9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Mar 2018 11:25:57 +0100
Subject: bin/build-packages: straw ":with_/dev/fuse:" new to pass "-d
 /dev/fuse"

---
 bin/build-packages | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/bin/build-packages b/bin/build-packages
index 70a0c3a..0f0c841 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -345,14 +345,23 @@ while [ "${count}" -ne 0 ]; do
 
         if echo "${straw}" | \
           grep -qF ':clean_chroot:'; then
-          parameters='-c'
+          outerParameters='-c'
         else
-          parameters=''
+          outerParameters=''
         fi
 
         if echo "${straw}" | \
           grep -qF ':without_check:'; then
-          parameters="${parameters} -- -- --nocheck"
+          innerParameters='--nocheck'
+        else
+          innerParameters=''
+        fi
+
+        if echo "${straw}" | \
+          grep -qF ':with_/dev/fuse:'; then
+          middleParameters='-d /dev/fuse'
+        else
+          middleParameters=''
         fi
 
         find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \
@@ -368,7 +377,7 @@ while [ "${count}" -ne 0 ]; do
         # by piping the log, we don't see anything in the terminal,
         # but all ways to duplicate the logs seem pretty elaborate
         # shellcheck disable=SC2024,SC2086
-        if sudo "${build_command}" ${parameters} > \
+        if sudo "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \
           "$(
             date -u --iso-8601=seconds | \
               cut -d+ -f1
-- 
cgit v1.2.3-54-g00ecf


From 5a5ef4ff52a57d60fed6f0f13e939f1c0103cf34 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Mar 2018 11:31:38 +0100
Subject: bin/build-packages: pass "--holdver" to makepkg, so cvs sources won't
 receive an update breaking our sub_pkgrel

---
 bin/build-packages | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/build-packages b/bin/build-packages
index 0f0c841..24c5453 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -352,9 +352,9 @@ while [ "${count}" -ne 0 ]; do
 
         if echo "${straw}" | \
           grep -qF ':without_check:'; then
-          innerParameters='--nocheck'
+          innerParameters='--nocheck --holdver'
         else
-          innerParameters=''
+          innerParameters='--holdver'
         fi
 
         if echo "${straw}" | \
-- 
cgit v1.2.3-54-g00ecf


From 148d6bbbdee8316cf1ea642ce8615e7c534f6019 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Mar 2018 11:49:50 +0100
Subject: lib/common-functions: PKGBUILD is r/o, so pkgver() won't change the
 version :-/

---
 bin/build-packages   | 4 ++--
 lib/common-functions | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/build-packages b/bin/build-packages
index 24c5453..0f0c841 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -352,9 +352,9 @@ while [ "${count}" -ne 0 ]; do
 
         if echo "${straw}" | \
           grep -qF ':without_check:'; then
-          innerParameters='--nocheck --holdver'
+          innerParameters='--nocheck'
         else
-          innerParameters='--holdver'
+          innerParameters=''
         fi
 
         if echo "${straw}" | \
diff --git a/lib/common-functions b/lib/common-functions
index ba434ad..0779203 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -934,6 +934,10 @@ extract_source_directory() {
   # shellcheck disable=SC2016
   sed -i '/^\$Id\$$/d' "${output}/PKGBUILD"
 
+  # we don't want write permissions on the PKGBUILD - otherwise pkgver()
+  # will change the version! (**HACK**)
+  chmod -w "${output}/PKGBUILD"
+
 }
 
 # find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository
-- 
cgit v1.2.3-54-g00ecf


From 8a24ee22bd08dff3f9610a837eaa0aacd7c6032a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Mar 2018 13:18:30 +0100
Subject: lib/mysql-functions: new joins for `repository_moves`

---
 lib/mysql-functions | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index a84d74c..9d705d2 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -1100,6 +1100,8 @@ for link in \
   'package_sources:upstream_package_repository:upstream_repositories' \
  \
   'repositories:stability:repository_stabilities' \
+ \
+  'repository_moves:upstream_package_repository:upstream_repositories' \
  \
   'upstream_repositories:git_repository:git_repositories'; do
 # A join for these cannot be done, because it's not clear on what to join:
-- 
cgit v1.2.3-54-g00ecf


From 390561a3f13eedec674259bb5b2e491886bb56d0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Mar 2018 16:15:09 +0100
Subject: bin/bootstrap-mysql: implement stored function with new condition
 from db-update (which is not yet used, but looks ok)

---
 bin/bootstrap-mysql | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 bin/db-update       |   6 +++
 2 files changed, 126 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index c828809..1e5f45f 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -301,6 +301,126 @@ if [ ! "$1" = 'slim' ]; then
     printf '//\n'
     printf 'DELIMITER ;\n'
 
+    # calculate_maximal_moveable_set
+    # stores results in `moveable_binary_packages` and `replaced_binary_packages`
+    # Give a maximal list of packages to be moved, while implementing the
+    # condition from db-update:
+    # Every package which is replaced, must have its provided install_targets:
+    #   a) provided by another moved or not-replaced package or
+    #   b) not required by any not-replaced package.
+    printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n'
+    printf 'DELIMITER //\n'
+    printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n'
+    printf 'BEGIN\n'
+    for table in 'moveable' 'replaced'; do
+      for copy in '' '_copy'; do
+        printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \
+          "${table}" "${copy}"
+        printf 'CREATE TEMPORARY TABLE `%s_binary_packages%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \
+          "${table}" "${copy}"
+      done
+    done
+    # these packages are considered for moving:
+    printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`)'
+    printf ' SELECT `binary_packages`.`id`'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    # correct stability: "testing"/"staging" - as chosen
+    printf ' WHERE `repository_stabilities`.`name` = `from_stability`'
+    printf ' AND ('
+      printf '`from_stability`="staging"'
+      # "testing" packages must be tested
+      printf ' OR `binary_packages`.`is_tested`'
+    printf ')'
+    # no open issues
+    printf ' AND NOT `binary_packages`.`has_issues`'
+    printf ';\n'
+    # these packages are considered for being replaced:
+    # for each moved package
+    printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`)'
+    printf ' SELECT `r_bp`.`id`'
+    printf ' FROM `moveable_binary_packages`'
+    printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`'
+    mysql_join_binary_packages_repositories 'm_bp' 'm_r'
+    mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba'
+    mysql_join_build_assignments_package_sources 'm_ba' 'm_ps'
+    mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur'
+    mysql_join_upstream_repositories_repository_moves 'm_ur'
+    printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`'
+    # in its target repository
+    printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`'
+    mysql_join_repositories_binary_packages 'r_r' 'r_bp'
+    # all packages with identical names
+    printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`;\n'
+    # now we delete all unreplaceable and unmoveable packages from the respective
+    # list until no further changes are required
+    printf 'REPEAT\n'
+    for s in 'replaced' 'moveable'; do
+      printf 'DELETE FROM `%s_binary_packages_copy`;\n' "$s"
+      printf 'INSERT IGNORE INTO `%s_binary_packages_copy` (`id`)' "$s"
+      printf ' SELECT `%s_binary_packages`.`id`' "$s"
+      printf ' FROM `%s_binary_packages`;\n' "$s"
+    done
+    # packages which should not be replaced:
+    printf 'DELETE `replaced_binary_packages` FROM `replaced_binary_packages`'
+    printf ' JOIN `binary_packages` AS `d_bp`'
+    mysql_join_binary_packages_dependencies 'd_bp'
+    mysql_join_binary_packages_repositories 'd_bp' 'd_r'
+    # consider only dependencies which can be met at all
+    printf ' AND EXISTS ('
+      printf 'SELECT * FROM `install_target_providers`'
+      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+    printf ')'
+    mysql_join_dependencies_dependency_types
+    # only consider runtime dependencies
+    printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+    # not provided by a not-replaced, not-less-stable package
+    printf ' AND NOT EXISTS ('
+      printf 'SELECT * FROM `install_target_providers`'
+      mysql_join_install_target_providers_binary_packages '' 'itp_bp'
+      mysql_join_binary_packages_repositories 'itp_bp' 'itp_r'
+      printf ' JOIN `repository_stability_relations` ON `itp_r`.`stability`=`repository_stability_relations`.`more_stable`'
+      printf ' WHERE `d_r`.`stability`=`repository_stability_relations`.`less_stable`'
+      printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+      printf ' AND NOT EXISTS ('
+        printf 'SELECT * FROM `replaced_binary_packages_copy`'
+        printf ' WHERE `replaced_binary_packages_copy`.`id`=`itp_bp`.`id`'
+      printf ')'
+    printf ')'
+    # not provided by a moved package
+    printf ' AND NOT EXISTS ('
+      printf 'SELECT * FROM `moveable_binary_packages`'
+      mysql_join_binary_packages_install_target_providers 'moveable_binary_packages'
+      printf ' WHERE `moveable_binary_packages`.`id`=`install_target_providers`.`package`'
+    printf ');\n'
+    # remove packages from the moveable-list which replace packages no longer on the replaced-list (e.g un-replaceable packages)
+    printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`'
+    printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`'
+    mysql_join_binary_packages_repositories 'm_bp' 'm_r'
+    mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba'
+    mysql_join_build_assignments_package_sources 'm_ba' 'm_ps'
+    mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur'
+    mysql_join_upstream_repositories_repository_moves 'm_ur'
+    printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`'
+    printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`'
+    mysql_join_repositories_binary_packages 'r_r' 'r_bp'
+    printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`'
+    printf ' WHERE NOT EXISTS ('
+      printf 'SELECT * FROM `replaced_binary_packages`'
+      printf ' WHERE `replaced_binary_packages`.`id`=`r_bp`.`id`'
+    printf ');\n'
+    # actually, this only checks the count of deletions from the moveable-list, but that should be ok
+    printf 'UNTIL ROW_COUNT()=0\n'
+    printf 'END REPEAT;\n'
+    for table in 'moveable' 'replaced'; do
+      printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy`;\n' \
+        "${table}"
+    done
+    printf 'END\n'
+    printf '//\n'
+    printf 'DELIMITER ;\n'
+
     printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \
       'CREATE ROUTINE' 'buildmaster.*' \
       'CREATE TEMPORARY TABLES' 'buildmaster.*' \
diff --git a/bin/db-update b/bin/db-update
index 78ff622..06754bb 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -10,6 +10,12 @@
 #   b) no done package B (in a not-more stable repository) which is
 #      not being moved run-depends on A
 
+# The new condition will be shorter, cleaner and less error prone:
+# (_Moving_ is without risk. It is _deleting_, that may break stuff.)
+# Every package which is replaced, must have its provided install_targets:
+#   a) provided by another new or not-replaced package or
+#   b) not required by any not-replaced package.
+
 # TODO: separate locks for staging, testing (and stable)
 
 # TODO: handle deletion of parts of a split package
-- 
cgit v1.2.3-54-g00ecf


From cca041f6592ac47a36937cbdf48f943c86c2d217 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Mar 2018 16:30:21 +0100
Subject: bin/db-update: add TODO hinting to stored mysql function

---
 bin/db-update | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index 06754bb..5efa220 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -16,6 +16,10 @@
 #   a) provided by another new or not-replaced package or
 #   b) not required by any not-replaced package.
 
+# TODO: Use the stored function
+# calculate_maximal_moveable_set("staging") and
+# calculate_maximal_moveable_set("testing")
+
 # TODO: separate locks for staging, testing (and stable)
 
 # TODO: handle deletion of parts of a split package
-- 
cgit v1.2.3-54-g00ecf


From 509cb31ae54326d3f8e0a5d9c42f5ef71aa7651b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 5 Mar 2018 19:43:07 +0100
Subject: lib/common-functions: remove pkgver() from PKGBUILD, so we do not
 accidentally update $pkgver

---
 lib/common-functions | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/common-functions b/lib/common-functions
index 0779203..0f75cb8 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -929,6 +929,10 @@ extract_source_directory() {
       "${output}/PKGBUILD"
   fi
 
+  # we do not want to update pkgver, so we just undefine it
+  printf 'unset -f pkgver\n' >> \
+    "${output}/PKGBUILD"
+
   mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}"
 
   # shellcheck disable=SC2016
-- 
cgit v1.2.3-54-g00ecf


From b90c2a7164a911a0fc30230f2817cbc3a65b4a86 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 6 Mar 2018 14:44:57 +0100
Subject: bin/interpret-mail: accept "\" as line break escaper

---
 bin/interpret-mail | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/interpret-mail b/bin/interpret-mail
index 1cc466e..bdcf016 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -168,7 +168,7 @@ sed -n '
         N
         bstart_loop
       }
-    s/=\s*\n//g
+    s/[=\]\s*\n//g
     s/:\s*\n/: /g
     s/\n\(\S\+[^: ]\(\s\|\n\|$\)\)/ \1/g
   ' > \
-- 
cgit v1.2.3-54-g00ecf


From 98c468dee44e09a27c89019c3d8c584efe0bf1c0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Mar 2018 16:29:58 +0100
Subject: bin/strict-bashism-check: do not fail if shellcheck is not available

---
 bin/strict-bashism-check | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/bin/strict-bashism-check b/bin/strict-bashism-check
index b1bd3bf..fa073b8 100755
--- a/bin/strict-bashism-check
+++ b/bin/strict-bashism-check
@@ -72,14 +72,16 @@ if [ -n "${errors}" ]; then
   exit 1
 fi
 
-errors=$(
-  cd "${tmp_dir}" || \
-    echo 'Cannot cd.'
-  shellcheck -x bin/* conf/* 2>&1
-)
-
-if [ -n "${errors}" ]; then
-  >&2 echo 'shellcheck complains about the following:'
-  >&2 echo "${errors}"
-  exit 1
+if which shellcheck >/dev/null 2>&1; then
+  errors=$(
+    cd "${tmp_dir}" || \
+      echo 'Cannot cd.'
+    shellcheck -x bin/* conf/* 2>&1
+  )
+
+  if [ -n "${errors}" ]; then
+    >&2 echo 'shellcheck complains about the following:'
+    >&2 echo "${errors}"
+    exit 1
+  fi
 fi
-- 
cgit v1.2.3-54-g00ecf


From a13bfbacf27d09f7c8ea6b4b559b0980ee8957b9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Mar 2018 16:30:10 +0100
Subject: make -N --raw --batch the default arguments to mysql, explicitely
 override them if required

---
 bin/build-master-status            |  2 +-
 bin/build-master-status-from-mysql | 16 ++++++++--------
 bin/check-bugtracker               |  2 +-
 bin/copy-to-build-support          |  2 +-
 bin/db-update                      |  4 ++--
 bin/get-assignment                 |  8 ++++----
 bin/get-package-updates            |  4 ++--
 bin/modify-package-state           |  2 +-
 bin/return-assignment              |  8 ++++----
 bin/seed-build-list                |  6 +++---
 bin/show-dependencies              |  4 ++--
 bin/why-dont-you                   |  2 +-
 lib/common-functions               |  2 +-
 lib/mysql-functions                | 18 +++++++++---------
 14 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index deb58da..006750b 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -562,7 +562,7 @@ if ${web}; then
     # shellcheck disable=SC2016
     {
       printf 'SHOW CREATE TABLE `todos`' | \
-        mysql_run_query -N --raw --batch | \
+        mysql_run_query | \
         sed '
           1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
         '
diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 7cf1b95..a72d396 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -27,7 +27,7 @@ fi
   mysql_join_binary_packages_repositories
   printf ' WHERE `repositories`.`name`="build-list"'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   tr '\t' ' ' | \
   sort > \
   "${tmp_dir}/build-list.mysql"
@@ -45,7 +45,7 @@ sort "${work_dir}/build-list" > \
   mysql_join_build_assignments_package_sources
   mysql_join_package_sources_upstream_repositories
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   tr '\t' ' ' | \
   sort > \
   "${tmp_dir}/locked.mysql"
@@ -66,7 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin
   printf ' WHERE `repositories`.`name`="deletion-list"'
   printf ' OR `binary_packages`.`is_to_be_deleted`;\n'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   sort > \
   "${tmp_dir}/deletion-list.mysql"
 sort "${work_dir}/deletion-list" > \
@@ -80,7 +80,7 @@ sort "${work_dir}/deletion-list" > \
   mysql_join_build_assignments_build_dependency_loops
   printf ';\n'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   sort > \
   "${tmp_dir}/loops.mysql"
 find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
@@ -107,7 +107,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   sed '
     s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
   ' | \
@@ -141,7 +141,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   printf ' AND NOT `binary_packages`.`has_issues`'
   printf ' AND `repository_stabilities`.`name`="testing"'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   sed '
     s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
   ' | \
@@ -174,7 +174,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   mysql_join_package_sources_upstream_repositories
   printf ' WHERE `dependency_types`.`relevant_for_building`'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   tr '\t' ' ' | \
   sort -u > \
   "${tmp_dir}/build-dependencies.mysql"
@@ -191,7 +191,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     mysql_join_package_sources_build_assignments
     mysql_join_build_assignments_binary_packages
   } | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     tr '\t' ' ' | \
     while read -r pkgbase git_revision mod_git_revision repository; do
       if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then
diff --git a/bin/check-bugtracker b/bin/check-bugtracker
index 156f438..5bd05b4 100755
--- a/bin/check-bugtracker
+++ b/bin/check-bugtracker
@@ -26,7 +26,7 @@ bug_list=$(
   printf ' FROM `repository_stabilities`'
   printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   while read -r stability_id category; do
     for has_issues in '1:' '0:NOT '; do
       printf 'UPDATE `binary_packages`'
diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index e447596..b3decd4 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -44,7 +44,7 @@ sed -n '
           "$(printf '%s' "${architecture}" | base64 -w0)"
         printf ' LIMIT 1;\n'
       } | \
-        mysql_run_query -N --raw --batch | \
+        mysql_run_query | \
         tr '\t' ' '
     )
     if [ -z "${id}" ]; then
diff --git a/bin/db-update b/bin/db-update
index 5efa220..5756494 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -467,7 +467,7 @@ errors=$(
       printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
       printf ' WHERE `repository_stabilities`.`name` = "testing"'
     } | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       sed 'p'
     # shellcheck disable=SC2086
     printf '%s\n' ${packages_to_force_stabilize} | \
@@ -503,7 +503,7 @@ errors=$(
       printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
       printf ' WHERE `repository_stabilities`.`name` = "staging"'
     } | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       sed 'p'
     # shellcheck disable=SC2086
     printf '%s\n' ${packages_to_force_unstage} | \
diff --git a/bin/get-assignment b/bin/get-assignment
index 964c6b7..2df3059 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -58,7 +58,7 @@ hand_out_assignment() {
       "$(printf '%s' "$4" | base64 -w0)"
     printf ' LIMIT 1;\n'
   } | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     tr '\t' ' '
 
   {
@@ -154,7 +154,7 @@ currently_building=$(
     printf ' AND `repositories`.`name`="build-list"'
     printf ' LIMIT 1;\n'
   } | \
-    mysql_run_query -N --batch --raw
+    mysql_run_query
 )
 
 if [ -n "${currently_building}" ]; then
@@ -210,7 +210,7 @@ next_building=$(
     printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`'
     printf ' LIMIT 1;\n'
   } | \
-    mysql_run_query -N --batch --raw | \
+    mysql_run_query | \
     sed '
       y/\t/ /
       s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/
@@ -233,7 +233,7 @@ count_pending=$(
     printf ' AND `build_assignments`.`is_blocked` IS NULL'
     printf ';\n'
   } | \
-    mysql_run_query -N --batch --raw
+    mysql_run_query
 )
 
 if [ "${count_pending}" -eq 0 ]; then
diff --git a/bin/get-package-updates b/bin/get-package-updates
index e855b19..a95bd94 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -391,7 +391,7 @@ black_listed_new=$(
       printf ' FROM `package_sources`'
       printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
     } | \
-      mysql_run_query -N --raw --batch
+      mysql_run_query
   } | \
     sort -u
 )
@@ -442,7 +442,7 @@ while [ -n "${black_listed_new}" ]; do
           printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
         printf ');\n'
       } | \
-        mysql_run_query -N --raw --batch | \
+        mysql_run_query | \
         sort -u
     } | \
       grep -vxF '' | \
diff --git a/bin/modify-package-state b/bin/modify-package-state
index 68b2384..9b0571b 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -225,7 +225,7 @@ while read -r package reason; do
     ;;
   esac
   if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     grep -qxF '1'; then
     # shellcheck disable=SC2016
     {
diff --git a/bin/return-assignment b/bin/return-assignment
index a7e6623..717a30e 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then
         "$(printf '%s' "$4" | base64 -w0)"
       printf ' AND `repositories`.`name`="build-list"'
     } | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       tr '\t' ' '
   )
   if [ -z "${infos}" ]; then
@@ -108,7 +108,7 @@ if [ "$5" = 'ERROR' ]; then
           printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")'
           printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`'
         } | \
-          mysql_run_query -N --raw --batch
+          mysql_run_query
       )
       for saved_build_log in ${saved_build_logs}; do
         printf '%s' "${fail_reason_identifiers}" | \
@@ -279,7 +279,7 @@ if ! {
     printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \
       "$(printf '%s' "$5" | base64 -w0)"
   } | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     grep -qvxF '0'; then
   >&2 echo 'Sorry, the sent package is outdated.'
   exit 2
@@ -436,7 +436,7 @@ package_errors=$(
       printf ' AND `repositories`.`name`="build-list"'
       printf ';\n'
     } | \
-      mysql_run_query -N --raw --batch
+      mysql_run_query
   } | \
     sort -k2 | \
     uniq -u -f1
diff --git a/bin/seed-build-list b/bin/seed-build-list
index def9d56..d49a8f3 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then
       printf ' FROM `binary_packages`'
       mysql_join_binary_packages_architectures
     } | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       sed '
         s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/
       '
@@ -205,7 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`'
   printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   while read -r pkgname; do
     content=$(
       curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
@@ -292,7 +292,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`'
   printf ');\n'
 } | \
-  mysql_run_query -N --raw --batch | \
+  mysql_run_query | \
   sort -u | \
   tr '\t' ' ' | \
   if ${update}; then
diff --git a/bin/show-dependencies b/bin/show-dependencies
index ed7ca87..492f11b 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -10,7 +10,7 @@
 if [ $# -eq 0 ]; then
   broken=$(
     printf 'CALL show_broken_packages_and_dependencies;\n' | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       sed '
         s/\s.*$//
       ' | \
@@ -177,7 +177,7 @@ for target_package in "$@"; do
     mysql_join_dependencies_install_targets
     printf ';\n'
   } | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     sed '
       y/\t/ /
       /^knot /{
diff --git a/bin/why-dont-you b/bin/why-dont-you
index 35b9419..64dfa1b 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -85,7 +85,7 @@ case "${action}" in
       printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`'
       printf ';\n'
     } | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       tr '\t' ' ' | \
       sort -k7,7 -k6,6 -k5,5 | \
       sed '
diff --git a/lib/common-functions b/lib/common-functions
index 0f75cb8..c7a64fe 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -501,7 +501,7 @@ remove_old_package_versions() {
       "$(printf '%s' "${repository}" | base64 -w0)"
     printf ';\n'
   } | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     tr '\t' ' ' | \
     expand_version 4 | \
     sort -k4V,4 -k3r,3 | \
diff --git a/lib/mysql-functions b/lib/mysql-functions
index 9d705d2..26d3db8 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -26,17 +26,17 @@ base64_encode_each() {
 # wrapper function to query mysql
 mysql_run_query() {
   if [ -s "${work_dir}/build-master-sanity" ]; then
-    ${mysql_command} "$@"
+    ${mysql_command} -N --raw --batch "$@"
   else
     {
-      printf '%s %s: ' "$0" "${mysql_command} $*"
+      printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*"
       date
     } \
       | tee -a "${work_dir}/mysql.stdin" \
       | tee -a "${work_dir}/mysql.stdout" \
       >> "${work_dir}/mysql.stderr"
     tee -a "${work_dir}/mysql.stdin" \
-      | ${mysql_command} "$@" \
+      | ${mysql_command} -N --raw --batch "$@" \
       2>> "${work_dir}/mysql.stderr" \
       | tee -a "${work_dir}/mysql.stdout"
     for s in \
@@ -193,7 +193,7 @@ mysql_show_binary_package() {
       s|, FROM| FROM|g
       s|AND;|;|g
     ' | \
-    mysql_run_query --html
+    mysql_run_query --html --column-names
 }
 
 # mysql_add_install_target $install_target
@@ -693,7 +693,7 @@ mysql_sanity_check() {
       'a' 'b'
     printf ';\n'
   } | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     sed '
       s,^,<font color="#FF0000">,
       s,$,</font>,
@@ -723,7 +723,7 @@ mysql_sanity_check() {
       mysql_join_binary_packages_repositories
       printf ' WHERE `repositories`.`is_on_master_mirror`'
     } | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       sed '
         s,\t,/,
         s,\t,-,
@@ -753,7 +753,7 @@ mysql_find_build_assignment_loops() {
       mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent'
       printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
     } | \
-      mysql_run_query -N --raw --batch | \
+      mysql_run_query | \
       tr '\t' ' ' | \
       tsort 2>&1 >/dev/null | \
       sed 's/^tsort:\s*//' | \
@@ -930,7 +930,7 @@ mysql_repair_binary_packages_without_build_assignment() {
     mysql_join_binary_packages_architectures
     printf ' WHERE `binary_packages`.`build_assignment`<0'
   } | \
-    mysql_run_query -N --raw --batch | \
+    mysql_run_query | \
     while read -r id pkgname arch; do
       pkgname=$(
         printf '%s' "${pkgname}" | \
@@ -958,7 +958,7 @@ mysql_repair_binary_packages_without_build_assignment() {
           printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
             "$(printf '%s' "${pkgname}" | base64 -w0)"
         } | \
-          mysql_run_query -N --raw --batch | \
+          mysql_run_query | \
           grep -qvxF '0'; then
         pkgbase="${pkgname}"
       fi
-- 
cgit v1.2.3-54-g00ecf


From b61c5f236ced1841b84c7d24feedad500b685509 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Mar 2018 20:01:15 +0100
Subject: bin/get-package-updates: remove broken, blocked and locked marker
 files of build assignments which get deleted

---
 bin/get-package-updates | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index a95bd94..1e0502f 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -105,6 +105,9 @@ delete_package() {
   echo "$1" >> \
     "${work_dir}/deletion-list.new"
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
+  find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \
+    -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(blocked\|locked\)' \
+    -delete
   # TODO: Once we want to rely on the database for test_exclusion, we
   # need to run the command below unconditionally, but with some
   # changes, so we can easily revert.
-- 
cgit v1.2.3-54-g00ecf


From f92358c7654a251016f251519ddf863d9637fef3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 7 Mar 2018 20:12:26 +0100
Subject: bin/get-package-updates: actually remove "broken" marker files ,too

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1e0502f..ffcb078 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -106,7 +106,7 @@ delete_package() {
     "${work_dir}/deletion-list.new"
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
   find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \
-    -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(blocked\|locked\)' \
+    -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \
     -delete
   # TODO: Once we want to rely on the database for test_exclusion, we
   # need to run the command below unconditionally, but with some
-- 
cgit v1.2.3-54-g00ecf


From 7e0a82e87099700c990c64e75ffef6e6792153a4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 08:38:20 +0100
Subject: bin/bootstrap-mysql: stored function works, but far too slow

---
 bin/bootstrap-mysql | 124 ++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 86 insertions(+), 38 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 1e5f45f..41d2c73 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -313,7 +313,11 @@ if [ ! "$1" = 'slim' ]; then
     printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n'
     printf 'BEGIN\n'
     for table in 'moveable' 'replaced'; do
-      for copy in '' '_copy'; do
+      for copy in '' '_copy' '_copy2'; do
+        if [ "${table}" = 'moveable' ] && \
+          [ "${copy}" = '_copy2' ]; then
+          continue
+        fi
         printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \
           "${table}" "${copy}"
         printf 'CREATE TEMPORARY TABLE `%s_binary_packages%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \
@@ -356,55 +360,92 @@ if [ ! "$1" = 'slim' ]; then
     # now we delete all unreplaceable and unmoveable packages from the respective
     # list until no further changes are required
     printf 'REPEAT\n'
-    for s in 'replaced' 'moveable'; do
-      printf 'DELETE FROM `%s_binary_packages_copy`;\n' "$s"
-      printf 'INSERT IGNORE INTO `%s_binary_packages_copy` (`id`)' "$s"
-      printf ' SELECT `%s_binary_packages`.`id`' "$s"
-      printf ' FROM `%s_binary_packages`;\n' "$s"
+    for table in 'replaced' 'moveable'; do
+printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}"
+      for copy in '' '2'; do
+        if [ "${table}" = 'moveable' ] && \
+          [ "${copy}" = '2' ]; then
+          continue
+        fi
+        printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}"
+        printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s` (`id`)' "${table}" "${copy}"
+        printf ' SELECT `%s_binary_packages`.`id`' "${table}"
+        printf ' FROM `%s_binary_packages`;\n' "${table}"
+      done
     done
-    # packages which should not be replaced:
+    # packages which should not be replaced: ones providing something that is:
+    # a) still needed by a not-replaced package x "less stable" than the target repository and
+    # b) not provided by:
+    #   1) a moved package or
+    #   2) any current, not-replaced package in a repository more stable than x's repository
+    #
+    # Note, that this is not 100% clean from an academic point of view:
+    # It might require _downgrading_ of a package to keep fulfilling a dependency
+    # and it might require to do so _without_ any chance for the user to notice,
+    # because there may be more "dependencies" in the database than in the package files.
+    #
+    # However, in practice both should not happen.
+    #
     printf 'DELETE `replaced_binary_packages` FROM `replaced_binary_packages`'
-    printf ' JOIN `binary_packages` AS `d_bp`'
-    mysql_join_binary_packages_dependencies 'd_bp'
-    mysql_join_binary_packages_repositories 'd_bp' 'd_r'
-    # consider only dependencies which can be met at all
-    printf ' AND EXISTS ('
-      printf 'SELECT * FROM `install_target_providers`'
-      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+    printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`'
+    mysql_join_binary_packages_install_target_providers 'repl_bp'
+    mysql_join_install_target_providers_dependencies
+    printf ' AND NOT EXISTS ('
+      # dependencies of replaced packages don't matter
+      printf 'SELECT * FROM `replaced_binary_packages_copy`'
+      printf ' WHERE `replaced_binary_packages_copy`.`id`=`dependencies`.`dependent`'
     printf ')'
     mysql_join_dependencies_dependency_types
-    # only consider runtime dependencies
+    # consider only runtime dependencies
     printf ' AND `dependency_types`.`relevant_for_binary_packages`'
-    # not provided by a not-replaced, not-less-stable package
-    printf ' AND NOT EXISTS ('
-      printf 'SELECT * FROM `install_target_providers`'
-      mysql_join_install_target_providers_binary_packages '' 'itp_bp'
-      mysql_join_binary_packages_repositories 'itp_bp' 'itp_r'
-      printf ' JOIN `repository_stability_relations` ON `itp_r`.`stability`=`repository_stability_relations`.`more_stable`'
-      printf ' WHERE `d_r`.`stability`=`repository_stability_relations`.`less_stable`'
-      printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+    mysql_join_dependencies_binary_packages '' 'req_bp'
+    mysql_join_binary_packages_repositories 'repl_bp' 'repl_r'
+    mysql_join_binary_packages_repositories 'req_bp' 'req_r'
+    # dependent package is "less stable" than dependency
+    printf ' JOIN `repository_stability_relations` AS `repl_rr`'
+    printf ' ON `repl_rr`.`more_stable`=`repl_r`.`stability`'
+    printf ' AND `repl_rr`.`less_stable`=`req_r`.`stability`'
+    # a) ^
+    printf ' WHERE NOT EXISTS ('
+      # no moved package ...
+      printf 'SELECT *'
+      printf ' FROM `moveable_binary_packages`'
+      mysql_join_binary_packages_install_target_providers 'moveable_binary_packages' 'subst_itp'
+      # ... provides the same
+      printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`'
+    # b) 1) ^
+    printf ') AND NOT EXISTS ('
+      # no current package ...
+      printf 'SELECT *'
+      printf ' FROM `binary_packages` AS `subst_bp`'
+      mysql_join_binary_packages_install_target_providers 'subst_bp' 'subst_itp'
+      # ... in a repository ...
+      mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+      # ... more stable ...
+      printf ' JOIN `repository_stability_relations` AS `subst_rr`'
+      printf ' ON `subst_rr`.`more_stable`=`subst_r`.`stability`'
+      # ... than x's repository ...
+      printf ' WHERE `subst_rr`.`less_stable`=`repl_r`.`stability`'
       printf ' AND NOT EXISTS ('
-        printf 'SELECT * FROM `replaced_binary_packages_copy`'
-        printf ' WHERE `replaced_binary_packages_copy`.`id`=`itp_bp`.`id`'
+      # ... and which is not replaced ...
+        printf 'SELECT * FROM `replaced_binary_packages_copy2`'
+        printf ' WHERE `replaced_binary_packages_copy2`.`id`=`subst_bp`.`id`'
       printf ')'
-    printf ')'
-    # not provided by a moved package
-    printf ' AND NOT EXISTS ('
-      printf 'SELECT * FROM `moveable_binary_packages`'
-      mysql_join_binary_packages_install_target_providers 'moveable_binary_packages'
-      printf ' WHERE `moveable_binary_packages`.`id`=`install_target_providers`.`package`'
+      # ... and provides the same
+      printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`'
+      # b) 2) ^
     printf ');\n'
-    # remove packages from the moveable-list which replace packages no longer on the replaced-list (e.g un-replaceable packages)
+    # packages which should not be moved: ones which replace packages no
+    # longer on the replaced-list (e.g un-replaceable packages)
     printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`'
     printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`'
-    mysql_join_binary_packages_repositories 'm_bp' 'm_r'
     mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba'
     mysql_join_build_assignments_package_sources 'm_ba' 'm_ps'
     mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur'
     mysql_join_upstream_repositories_repository_moves 'm_ur'
-    printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`'
-    printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`'
-    mysql_join_repositories_binary_packages 'r_r' 'r_bp'
+    printf ' AND `repository_moves`.`from_repository`=`m_bp`.`repository`'
+    printf ' JOIN `binary_packages` AS `r_bp`'
+    printf ' ON `repository_moves`.`to_repository`=`r_bp`.`repository`'
     printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`'
     printf ' WHERE NOT EXISTS ('
       printf 'SELECT * FROM `replaced_binary_packages`'
@@ -414,8 +455,14 @@ if [ ! "$1" = 'slim' ]; then
     printf 'UNTIL ROW_COUNT()=0\n'
     printf 'END REPEAT;\n'
     for table in 'moveable' 'replaced'; do
-      printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy`;\n' \
-        "${table}"
+      for copy in '' '2'; do
+        if [ "${table}" = 'moveable' ] && \
+          [ "${copy}" = '2' ]; then
+          continue
+        fi
+        printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy%s`;\n' \
+          "${table}" "${copy}"
+      done
     done
     printf 'END\n'
     printf '//\n'
@@ -427,6 +474,7 @@ if [ ! "$1" = 'slim' ]; then
       'EXECUTE' 'buildmaster.*' \
       'RELOAD' '*.*' \
       'SELECT' 'buildmaster.*' \
+      'SELECT' 'mysql.proc' \
       'SHOW VIEW' 'buildmaster.*' \
       'UPDATE' 'buildmaster.*'
     printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \
-- 
cgit v1.2.3-54-g00ecf


From 344682e7a5bb10dc9b3eed360845223247525c9f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 08:38:48 +0100
Subject: bin/db-update: call calculate_maximal_moveable_set, but do nothing
 with the output

---
 bin/db-update | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index 5756494..9a5fb1f 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -564,6 +564,33 @@ for ending in 'done' 'tested'; do
   fi
 done
 
+{
+  printf 'CALL calculate_maximal_moveable_set("testing");\n'
+  printf 'SELECT "mv",'
+  mysql_package_name_query
+  printf ',`repositories`.`name`,`new_repo`.`name`'
+  printf ' FROM `moveable_binary_packages`'
+  printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  mysql_join_binary_packages_build_assignments
+  mysql_join_build_assignments_package_sources
+  mysql_join_package_sources_upstream_repositories
+  mysql_join_upstream_repositories_repository_moves
+  printf ' AND `repositories`.`id`=`repository_moves`.`from_repository`'
+  printf ' JOIN `repositories` AS `new_repo` ON `new_repo`.`id`=`repository_moves`.`to_repository`'
+  printf ';\n'
+  printf 'SELECT "rm",'
+  mysql_package_name_query
+  printf ',`repositories`.`name`'
+  printf ' FROM `replaced_binary_packages`'
+  printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  printf ';\n'
+} | \
+  mysql_run_query -N --raw --batch
+
 # packages which are done
 
 find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
-- 
cgit v1.2.3-54-g00ecf


From f1bf6de03ad68e75dd2602a1744cc30a4a23b6f5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 08:47:57 +0100
Subject: lib/mysql-functions: add comment

---
 lib/mysql-functions | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index 26d3db8..97fc2f4 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -26,6 +26,8 @@ base64_encode_each() {
 # wrapper function to query mysql
 mysql_run_query() {
   if [ -s "${work_dir}/build-master-sanity" ]; then
+  # If the build master is insane, the calling command should only check
+  # if the build master is still insane - we do not want to log that.
     ${mysql_command} -N --raw --batch "$@"
   else
     {
-- 
cgit v1.2.3-54-g00ecf


From 1402c0c7565e1a1ecc55c0383727de5b6ad0dba3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 09:06:01 +0100
Subject: bin/sanity-check: print into "messages" in case of insane database

---
 bin/sanity-check | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/sanity-check b/bin/sanity-check
index 601a940..9487726 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -395,7 +395,8 @@ while [ $# -gt 0 ]; do
         if [ ${silence} -le 1 ]; then
           printf '\nThere is something wrong with the database:\n'
           cat "${webserver_directory}/mysql-sanity.html"
-        fi
+        fi >> \
+          "${tmp_dir}/messages"
         i_am_insane
       fi
 
-- 
cgit v1.2.3-54-g00ecf


From 4a8a8b2408ebf2543bc9fb00165b502a12cef7a2 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 09:13:07 +0100
Subject: lib/mysql-functions: mysql_run_query: we save the query in a file and
 delete that file if the query succeeded

---
 lib/mysql-functions | 73 ++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 55 insertions(+), 18 deletions(-)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index 97fc2f4..d87b79e 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -25,33 +25,70 @@ base64_encode_each() {
 # mysql_run_query
 # wrapper function to query mysql
 mysql_run_query() {
+  local query_file
   if [ -s "${work_dir}/build-master-sanity" ]; then
   # If the build master is insane, the calling command should only check
   # if the build master is still insane - we do not want to log that.
     ${mysql_command} -N --raw --batch "$@"
   else
-    {
-      printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*"
-      date
-    } \
-      | tee -a "${work_dir}/mysql.stdin" \
-      | tee -a "${work_dir}/mysql.stdout" \
-      >> "${work_dir}/mysql.stderr"
-    tee -a "${work_dir}/mysql.stdin" \
-      | ${mysql_command} -N --raw --batch "$@" \
-      2>> "${work_dir}/mysql.stderr" \
-      | tee -a "${work_dir}/mysql.stdout"
-    for s in \
-      "${work_dir}/mysql.stdin" \
-      "${work_dir}/mysql.stdout" \
-      "${work_dir}/mysql.stderr"; do
+    # we save the query in a file and delete that file if the query succeeded
+    query_file=$(mktemp "${work_dir}/tmp.mysql-functions.query.$(date +'%Y-%m-%dT%T').XXXXXX")
+    cat > "${query_file}"
+    for i in {1..10}; do
       {
-        tail -n 10000 "$s"
-        printf '%s %s done: ' "$0" "${mysql_command} $*"
+        printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*"
         date
       } \
-        | sponge "$s"
+        | tee -a "${work_dir}/mysql.stdin" \
+        | tee -a "${work_dir}/mysql.stdout" \
+        >> "${work_dir}/mysql.stderr"
+      cat "${query_file}" \
+        >> "${work_dir}/mysql.stdin"
+      {
+        ${mysql_command} -N --raw --batch "$@" \
+          < "${query_file}" \
+          2>> "${work_dir}/mysql.stderr" \
+          && rm "${query_file}"
+      } \
+        | tee -a "${work_dir}/mysql.stdout"
+      if ! [ -f "${query_file}" ]; then
+        # success!
+        for s in \
+          "${work_dir}/mysql.stdin" \
+          "${work_dir}/mysql.stdout" \
+          "${work_dir}/mysql.stderr"; do
+          {
+            tail -n 10000 "$s"
+            printf '%s %s done: ' "$0" "${mysql_command} $*"
+            date
+          } \
+            | sponge "$s"
+        done
+        break
+      fi
+      for s in \
+        "${work_dir}/mysql.stdin" \
+        "${work_dir}/mysql.stdout" \
+        "${work_dir}/mysql.stderr"; do
+        {
+          printf '%s %s FAILED: ' "$0" "${mysql_command} $*"
+          date
+        } \
+          >> "$s"
+      done
     done
+    # a present query_file means there was an error
+    if [ -f "${query_file}" ]; then
+      >&2 printf 'I could not complete a mysql query!\n'
+      if [ ! -s "${work_dir}/build-master-sanity" ]; then
+        printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \
+          "${query_file##*/}" \
+          | sponge "${irc_dir}/#archlinux-ports/in"
+      fi
+      echo 'A mysql query failed.' > \
+        "${work_dir}/build-master-sanity"
+      return 2
+    fi
   fi
 }
 
-- 
cgit v1.2.3-54-g00ecf


From 5423c468f869116deb7de781373b8d35d61a136c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 09:14:30 +0100
Subject: bin/sanity-check: check for pending mysql queries

---
 bin/sanity-check | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bin/sanity-check b/bin/sanity-check
index 9487726..824b7a2 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -400,6 +400,12 @@ while [ $# -gt 0 ]; do
         i_am_insane
       fi
 
+      if find "${work_dir}" -mindepth 1 -maxdepth 1 -name 'tmp.mysql-functions.query.*' | \
+        grep '\S' >> \
+        "${tmp_dir}/messages"; then
+        i_am_insane
+      fi
+
       [ ${silence} -gt 0 ] || \
         echo ' passed.' >> \
         "${tmp_dir}/messages"
-- 
cgit v1.2.3-54-g00ecf


From 528597cafd46da1122299e1a73e7360c46b01014 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 09:36:17 +0100
Subject: bin/db-update: cleanup mysql database before accessing it (to
 accellerate database operations)

---
 bin/db-update | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index 9a5fb1f..8d2c0b5 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -548,6 +548,9 @@ clean_up_lock_file() {
 
 trap clean_up_lock_file EXIT
 
+# shellcheck disable=SC2119
+mysql_cleanup
+
 # sanity check
 
 for ending in 'done' 'tested'; do
@@ -564,6 +567,7 @@ for ending in 'done' 'tested'; do
   fi
 done
 
+# shellcheck disable=SC2016
 {
   printf 'CALL calculate_maximal_moveable_set("testing");\n'
   printf 'SELECT "mv",'
-- 
cgit v1.2.3-54-g00ecf


From aa97f3e03393b01bf9df9a8e4f6e7f3b09c7905c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 13:11:02 +0100
Subject: bin/bootstrap-mysql: calculate_maximal_moveable_set: have more
 columns in temporary tables to avoid some querys/joins

---
 bin/bootstrap-mysql | 62 ++++++++++++++++++++++-------------------------------
 1 file changed, 26 insertions(+), 36 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 41d2c73..4e118ee 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -312,24 +312,29 @@ if [ ! "$1" = 'slim' ]; then
     printf 'DELIMITER //\n'
     printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n'
     printf 'BEGIN\n'
-    for table in 'moveable' 'replaced'; do
-      for copy in '' '_copy' '_copy2'; do
-        if [ "${table}" = 'moveable' ] && \
-          [ "${copy}" = '_copy2' ]; then
-          continue
-        fi
-        printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \
-          "${table}" "${copy}"
-        printf 'CREATE TEMPORARY TABLE `%s_binary_packages%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \
-          "${table}" "${copy}"
-      done
+    for copy in '' '_copy' '_copy2'; do
+      printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \
+        'moveable' "${copy}" \
+        'replaced' "${copy}"
+      printf 'CREATE TEMPORARY TABLE `replaced_binary_packages%s` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' \
+        "${copy}"
+      if [ "${copy}" = '_copy2' ]; then
+        continue
+      fi
+      printf 'CREATE TEMPORARY TABLE `moveable_binary_packages%s` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' \
+        "${copy}"
     done
     # these packages are considered for moving:
-    printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`)'
-    printf ' SELECT `binary_packages`.`id`'
+    printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+    printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`'
     printf ' FROM `binary_packages`'
     mysql_join_binary_packages_repositories
     mysql_join_repositories_repository_stabilities
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
+    mysql_join_upstream_repositories_repository_moves
+    printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
     # correct stability: "testing"/"staging" - as chosen
     printf ' WHERE `repository_stabilities`.`name` = `from_stability`'
     printf ' AND ('
@@ -342,8 +347,8 @@ if [ ! "$1" = 'slim' ]; then
     printf ';\n'
     # these packages are considered for being replaced:
     # for each moved package
-    printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`)'
-    printf ' SELECT `r_bp`.`id`'
+    printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+    printf ' SELECT `r_bp`.`id`,`m_bp`.`id`'
     printf ' FROM `moveable_binary_packages`'
     printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`'
     mysql_join_binary_packages_repositories 'm_bp' 'm_r'
@@ -361,15 +366,15 @@ if [ ! "$1" = 'slim' ]; then
     # list until no further changes are required
     printf 'REPEAT\n'
     for table in 'replaced' 'moveable'; do
-printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}"
+printf 'SELECT NOW(),"count(%s)",count(*) FROM `%s_binary_packages`;\n' "${table}" "${table}"
       for copy in '' '2'; do
         if [ "${table}" = 'moveable' ] && \
           [ "${copy}" = '2' ]; then
           continue
         fi
         printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}"
-        printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s` (`id`)' "${table}" "${copy}"
-        printf ' SELECT `%s_binary_packages`.`id`' "${table}"
+        printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}"
+        printf ' SELECT `%s_binary_packages`.*' "${table}"
         printf ' FROM `%s_binary_packages`;\n' "${table}"
       done
     done
@@ -386,7 +391,8 @@ printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}"
     #
     # However, in practice both should not happen.
     #
-    printf 'DELETE `replaced_binary_packages` FROM `replaced_binary_packages`'
+    printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`'
+    printf ' JOIN `moveable_binary_packages` ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`'
     printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`'
     mysql_join_binary_packages_install_target_providers 'repl_bp'
     mysql_join_install_target_providers_dependencies
@@ -399,6 +405,7 @@ printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}"
     # consider only runtime dependencies
     printf ' AND `dependency_types`.`relevant_for_binary_packages`'
     mysql_join_dependencies_binary_packages '' 'req_bp'
+    # we need to check wether req_bp's dependency is (un)critical
     mysql_join_binary_packages_repositories 'repl_bp' 'repl_r'
     mysql_join_binary_packages_repositories 'req_bp' 'req_r'
     # dependent package is "less stable" than dependency
@@ -435,23 +442,6 @@ printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}"
       printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`'
       # b) 2) ^
     printf ');\n'
-    # packages which should not be moved: ones which replace packages no
-    # longer on the replaced-list (e.g un-replaceable packages)
-    printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`'
-    printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`'
-    mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba'
-    mysql_join_build_assignments_package_sources 'm_ba' 'm_ps'
-    mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur'
-    mysql_join_upstream_repositories_repository_moves 'm_ur'
-    printf ' AND `repository_moves`.`from_repository`=`m_bp`.`repository`'
-    printf ' JOIN `binary_packages` AS `r_bp`'
-    printf ' ON `repository_moves`.`to_repository`=`r_bp`.`repository`'
-    printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`'
-    printf ' WHERE NOT EXISTS ('
-      printf 'SELECT * FROM `replaced_binary_packages`'
-      printf ' WHERE `replaced_binary_packages`.`id`=`r_bp`.`id`'
-    printf ');\n'
-    # actually, this only checks the count of deletions from the moveable-list, but that should be ok
     printf 'UNTIL ROW_COUNT()=0\n'
     printf 'END REPEAT;\n'
     for table in 'moveable' 'replaced'; do
-- 
cgit v1.2.3-54-g00ecf


From eddb9d9e1d01c15f48366e5210f90f51a4d38c32 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 13:15:23 +0100
Subject: bin/bootstrap-mysql: deduplicate names

---
 bin/bootstrap-mysql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 4e118ee..8b1db28 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -416,8 +416,8 @@ printf 'SELECT NOW(),"count(%s)",count(*) FROM `%s_binary_packages`;\n' "${table
     printf ' WHERE NOT EXISTS ('
       # no moved package ...
       printf 'SELECT *'
-      printf ' FROM `moveable_binary_packages`'
-      mysql_join_binary_packages_install_target_providers 'moveable_binary_packages' 'subst_itp'
+      printf ' FROM `moveable_binary_packages_copy`'
+      mysql_join_binary_packages_install_target_providers 'moveable_binary_packages_copy' 'subst_itp'
       # ... provides the same
       printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`'
     # b) 1) ^
-- 
cgit v1.2.3-54-g00ecf


From acfd4fd009f2dc080688c44cb0e1070083a62f50 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 15:34:00 +0100
Subject: bin/bootstrap-mysql: remove debug output

---
 bin/bootstrap-mysql | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 8b1db28..8b01c38 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -366,7 +366,6 @@ if [ ! "$1" = 'slim' ]; then
     # list until no further changes are required
     printf 'REPEAT\n'
     for table in 'replaced' 'moveable'; do
-printf 'SELECT NOW(),"count(%s)",count(*) FROM `%s_binary_packages`;\n' "${table}" "${table}"
       for copy in '' '2'; do
         if [ "${table}" = 'moveable' ] && \
           [ "${copy}" = '2' ]; then
-- 
cgit v1.2.3-54-g00ecf


From f86e8f204c54c5354c5db44c62ae3517a765d202 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 8 Mar 2018 18:18:57 +0100
Subject: bin/bootstrap-mysql: add todo

---
 bin/bootstrap-mysql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 1e5f45f..519b493 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -32,6 +32,8 @@ done | \
   sort -k2,2 > \
   "${tmp_dir}/master-mirror-listing"
 
+# TODO: include hash of binary package in mysql database
+
 if [ ! "$1" = 'slim' ]; then
   tables=$(
     printf '%s\n' \
-- 
cgit v1.2.3-54-g00ecf


From 32f570cbe02326c27b61faf82dbe040e33ed00f6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 09:06:09 +0100
Subject: bin/db-update: fully rely on database

---
 bin/db-update | 819 ++++++++++++----------------------------------------------
 1 file changed, 164 insertions(+), 655 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 8d2c0b5..a7daa7c 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -4,29 +4,14 @@
 # additionally tested packages from testing to the respective stable
 # repository (if possible [1])
 
-# 1] Condition for moving a package A from staging (testing) to
-#    testing (stable) is that:
-#   a) nothing on the build-list run-depends on A and
-#   b) no done package B (in a not-more stable repository) which is
-#      not being moved run-depends on A
-
-# The new condition will be shorter, cleaner and less error prone:
-# (_Moving_ is without risk. It is _deleting_, that may break stuff.)
-# Every package which is replaced, must have its provided install_targets:
-#   a) provided by another new or not-replaced package or
-#   b) not required by any not-replaced package.
-
-# TODO: Use the stored function
-# calculate_maximal_moveable_set("staging") and
-# calculate_maximal_moveable_set("testing")
+# The condition [1] is explained in the stored function
+# calculate_maximal_moveable_set which is created in bin/bootsrap-mysql
 
 # TODO: separate locks for staging, testing (and stable)
 
-# TODO: handle deletion of parts of a split package
+# TODO: sub_pkgrel=0 may or may not be omitted for the actual package :-/
 
-# TODO: handle dependencies of parts of a split package separately
-
-# TODO: read information from database
+# TODO: keep .testing, .tested, .done in sync
 
 # shellcheck disable=SC2039
 # shellcheck source=conf/default.conf
@@ -36,367 +21,21 @@
 usage() {
   >&2 echo ''
   >&2 echo 'db-update [options] [packages]:'
-  >&2 echo ' move possible packages from staging to testing.'
   >&2 echo ' move tested packages from testing to stable.'
+  >&2 echo ' move possible packages from staging to testing.'
   >&2 echo ''
   >&2 echo 'possible options:'
   >&2 echo '  -b|--block:       If necessary, wait for lock blocking.'
   >&2 echo '  -h|--help:        Show this help and exit.'
   >&2 echo '  -n|--no-action:   Only print what would be moved.'
-  >&2 echo '  -s|--stabilize $package:'
-  >&2 echo '                    Assume, package $package can be stabilized.'
-  >&2 echo '  -u|--unstage $package:'
-  >&2 echo '                    Assume, package $package can be unstaged.'
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
-# move_packages file with one "$package $from_repository $to_repository" per line
-# the existence of a directory $tmp_dir is assumed
-
-move_packages() {
-
-  if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
-    >&2 echo 'move_packages: No tmp_dir provided.'
-    exit 2
-  fi
-
-  local package
-  local from_repo
-  local to_repo
-  local from_ending
-  local rm_ending
-  local to_ending
-  local repo
-  local part
-  local dummynator
-  local file
-
-  if [ -e "${tmp_dir:?}/tmp" ]; then
-    rm -rf --one-file-system "${tmp_dir:?}/tmp"
-  fi
-  mkdir "${tmp_dir}/tmp"
-
-  touch "${tmp_dir}/tmp/repos"
-  touch "${tmp_dir}/tmp/packages"
-  touch "${tmp_dir}/tmp/master-mirror-listing"
-  mkdir "${tmp_dir}/tmp/transit"
-
-  if ${no_action}; then
-    dummynator='echo'
-  else
-    dummynator=''
-  fi
-
-  ls_master_mirror 'i686' | \
-    while read -r repo; do
-      ls_master_mirror "i686/${repo}" | \
-        sed "s|^|i686/${repo}/|" >> \
-        "${tmp_dir}/tmp/master-mirror-listing"
-    done
-
-  while read -r package from_repo to_repo; do
-    if [ -z "${package}" ]; then
-      continue
-    fi
-
-    if ${no_action}; then
-      printf \
-        'move "%s" from "%s" to "%s"\n' \
-        "${package}" \
-        "${from_repo}" \
-        "${to_repo}"
-    fi
-
-    echo "${package}" >> \
-      "${tmp_dir}/tmp/packages"
-
-    if echo "${from_repo}" | \
-      grep -q 'staging$' && \
-      echo "${to_repo}" | \
-        grep -q 'testing$'; then
-      from_ending='done'
-      rm_ending='test\(ing\|ed\)'
-      to_ending='testing'
-    elif echo "${from_repo}" | \
-      grep -q 'testing$' && \
-      ! echo "${to_repo}" | \
-        grep -q 'testing$\|staging$'; then
-      from_ending='tested'
-      rm_ending=''
-      to_ending=''
-    else
-      >&2 printf 'move_packages: Cannot move package "%s" from "%s" to "%s".\n' "${package}" "${from_repo}" "${to_repo}"
-      exit 2
-    fi
-
-    echo "${from_repo}" > \
-      "${tmp_dir}/tmp/${package}.from_repo"
-    echo "${to_repo}" > \
-      "${tmp_dir}/tmp/${package}.to_repo"
-    echo "${from_ending}" > \
-      "${tmp_dir}/tmp/${package}.from_ending"
-    echo "${rm_ending}" > \
-      "${tmp_dir}/tmp/${package}.rm_ending"
-    echo "${to_ending}" > \
-      "${tmp_dir}/tmp/${package}.to_ending"
-
-    if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
-      >&2 printf 'move_packages: Cannot find package state file "%s"\n' "${package}.${from_ending}"
-      exit 2
-    fi
-
-    cp \
-      "${work_dir}/package-states/${package}.${from_ending}" \
-      "${tmp_dir}/tmp/${package}.parts"
-
-    sed \
-      's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
-      "${tmp_dir}/tmp/${package}.parts" > \
-      "${tmp_dir}/tmp/${package}.parts_names"
-
-    sed \
-      'p;s|$|.sig|' \
-      "${tmp_dir}/tmp/${package}.parts" > \
-      "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
-    while read -r part; do
-      if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/tmp/master-mirror-listing"; then
-        >&2 printf \
-          'move_packages: Cannot find file "%s", part of package "%s".\n' \
-          "i686/${from_repo}/${part}" \
-          "${package}"
-        exit 2
-      fi
-    done < \
-      "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
-    mkdir -p "${tmp_dir}/tmp/${from_repo}"
-    mkdir -p "${tmp_dir}/tmp/${to_repo}"
-
-    repos=$(
-      # shellcheck disable=SC2046
-      printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/tmp/repos") | \
-        sort -u
-    )
-    echo "${repos}" > \
-      "${tmp_dir}/tmp/repos"
-
-  done < \
-    "$1"
-
-  if ${no_action}; then
-    find "${tmp_dir}/tmp" -type f | \
-      while read -r file; do
-        if [ "${file%.pkg.tar.xz}.pkg.tar.xz" = "${file}" ] ||
-          [ "${file%.pkg.tar.xz.sig}.pkg.tar.xz.sig" = "${file}" ]; then
-          echo "'${file}'"
-        else
-          echo "${file}:"
-          sed 's|^|<<|;s|$|>>|' "${file}"
-        fi
-        echo
-      done
-  fi
-
-  # receive the *.db.tar.gz's and *.files.tar.gz's
-
-  while read -r repo; do
-
-    ${master_mirror_rsync_command} \
-      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
-      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
-      "${tmp_dir}/tmp/${repo}/"
-
-    # add and remove the packages locally
-
-    if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".from_repo"; then
-
-      # shellcheck disable=SC2046
-      repo-remove -q \
-        "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
-        $(
-          grep -lxF "${repo}" "${tmp_dir}/tmp/"*".from_repo" | \
-            sed '
-              s|\.from_repo$|.parts_names|
-            ' | \
-            xargs -rn1 cat
-        )
-    fi
-
-    if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".to_repo"; then
-      grep -lxF "${repo}" "${tmp_dir}/tmp/"*".to_repo" | \
-        sed '
-          s|\.to_repo$||
-        ' | \
-        while read -r package; do
-          while read -r part; do
-            ${master_mirror_rsync_command} \
-              "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}" \
-              "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}.sig" \
-              "${tmp_dir}/tmp/transit/"
-            repo-add -q \
-              "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
-              "${tmp_dir}/tmp/transit/${part}"
-            rm \
-              "${tmp_dir}/tmp/transit/${part}" \
-              "${tmp_dir}/tmp/transit/${part}.sig"
-          done < \
-            "${package}.parts"
-        done
-    fi
-
-  done < "${tmp_dir}/tmp/repos"
-
-  if ${no_action}; then
-    find "${tmp_dir}/tmp" -type f
-  fi
-
-  # move the packages remotely via sftp
-
-  {
-    while read -r package; do
-
-      if [ -z "${package}" ]; then
-        continue
-      fi
-
-      while read -r part; do
-        if [ -z "${part}" ]; then
-          continue
-        fi
-        printf \
-          'rename "%s" "%s"\n' \
-          "i686/$(cat "${tmp_dir}/tmp/${package}.from_repo")/${part}" \
-          "i686/$(cat "${tmp_dir}/tmp/${package}.to_repo")/${part}"
-      done < \
-        "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
-    done < \
-      "${tmp_dir}/tmp/packages"
-    echo 'quit'
-  } | \
-    if ${no_action}; then
-      sed 's|^|sftp: |'
-    else
-      ${master_mirror_sftp_command}
-    fi
-
-  # and push our local *.db.tar.gz via rsync
-
-  while read -r repo; do
-
-    # shellcheck disable=SC2086
-    ${dummynator} ${master_mirror_rsync_command} \
-      "${tmp_dir}/tmp/${repo}/${repo}.db."* \
-      "${tmp_dir}/tmp/${repo}/${repo}.files."* \
-      "${master_mirror_rsync_directory}/i686/${repo}/"
-
-  done < \
-    "${tmp_dir}/tmp/repos"
-
-  while read -r package; do
-
-    # then we can safely remove old versions
-
-    while read -r part; do
-      ${dummynator} remove_old_package_versions 'i686' "$(cat "${tmp_dir}/tmp/${package}.to_repo")" "${part}"
-    done < \
-      "${tmp_dir}/tmp/${package}.parts"
-
-    # and update the state files
-
-    from_ending=$(
-      cat "${tmp_dir}/tmp/${package}.from_ending"
-    )
-    rm_ending=$(
-      cat "${tmp_dir}/tmp/${package}.rm_ending"
-    )
-    to_ending=$(
-      cat "${tmp_dir}/tmp/${package}.to_ending"
-    )
-
-    if [ -z "${to_ending}" ]; then
-      ${dummynator} rm \
-        "${work_dir}/package-states/${package}.${from_ending}"
-    else
-      # remove old state files of $package with ending $rm_ending
-      find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
-        -regex '.*/'"$(str_to_regex "${package%.*.*.*}")"'\(\.[^.]\+\)\{3\}\.'"${rm_ending}" \
-        -execdir ${dummynator} rm {} \;
-      ${dummynator} mv \
-        "${work_dir}/package-states/${package}.${from_ending}" \
-        "${work_dir}/package-states/${package}.${to_ending}"
-    fi
-
-  done < \
-    "${tmp_dir}/tmp/packages"
-
-  # shellcheck disable=SC2016
-  while read -r package; do
-    while read -r part; do
-      printf 'UPDATE `binary_packages`'
-      mysql_join_binary_packages_repositories '' 'from_repo'
-      mysql_join_binary_packages_architectures
-      printf ' SET `binary_packages`.`repository`=('
-        printf 'SELECT `to_repo`.`id`'
-        printf ' FROM `repositories` as `to_repo`'
-        printf ' WHERE `to_repo`.`name`=from_base64("%s")' \
-          "$(
-            base64_encode_each < \
-              "${tmp_dir}/tmp/${package}.to_repo"
-          )"
-        printf ')'
-      printf ' WHERE'
-      printf ' `from_repo`.`name`=from_base64("%s")' \
-        "$(
-          base64_encode_each < \
-            "${tmp_dir}/tmp/${package}.from_repo"
-        )"
-      printf '%s\n' "${part}" | \
-        sed '
-          s/\.pkg\.tar\.xz$//
-          s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/
-          s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
-          s/^\(.\+\)-\([^-:]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/\1\n\2\n\3\n\4\n\5\n\6/
-        ' | \
-        base64_encode_each | \
-        tr '\n' ' ' | \
-        sed '
-          s,\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) $,'"$(
-            printf ' AND `binary_packages`.`%s`=from_base64("%s")' \
-              'pkgname' '\1' \
-              'epoch' '\2' \
-              'pkgver' '\3' \
-              'pkgrel' '\4' \
-              'sub_pkgrel' '\5'
-            printf ' AND `architectures`.`name`=from_base64("\\6")'
-          )"',
-        '
-      printf ';\n'
-    done < \
-      "${tmp_dir}/tmp/${package}.parts"
-  done < \
-    "${tmp_dir}/tmp/packages" | \
-    if ${no_action}; then
-      sed 's|^|mysql: |'
-    else
-      mysql_run_query
-    fi
-
-  ${dummynator} trigger_mirror_refreshs
-
-  rm -rf --one-file-system "${tmp_dir:?}/tmp"
-
-}
-
 eval set -- "$(
-  getopt -o bf:hns:u: \
+  getopt -o bhn \
     --long block \
-    --long from: \
     --long help \
     --long no-action \
-    --long stabilize: \
-    --long unstage: \
     -n "$(basename "$0")" -- "$@" || \
   echo usage
 )"
@@ -416,14 +55,6 @@ do
     -n|--no-action)
       no_action=true
     ;;
-    -s|--stabilze)
-      shift
-      packages_to_force_stabilize="${packages_to_force_stabilize} $1"
-    ;;
-    -u|--unstage)
-      shift
-      packages_to_force_unstage="${packages_to_force_unstage} $1"
-    ;;
     --)
       shift
       break
@@ -446,311 +77,189 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
+# Create tmp_dir, lock and trap.
+
 tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-errors=$(
-  {
-    # shellcheck disable=SC2016
-    {
-      printf 'SELECT DISTINCT CONCAT('
-      printf '`package_sources`.`%s`,".",' \
-        'pkgbase' 'git_revision' 'mod_git_revision'
-      printf '`upstream_repositories`.`name`)'
-      printf ' FROM `binary_packages`'
-      mysql_join_binary_packages_repositories
-      mysql_join_binary_packages_build_assignments
-      mysql_join_build_assignments_package_sources
-      mysql_join_package_sources_upstream_repositories
-      # consider all packages less stable than "testing"
-      printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
-      printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
-      printf ' WHERE `repository_stabilities`.`name` = "testing"'
-    } | \
-      mysql_run_query | \
-      sed 'p'
-    # shellcheck disable=SC2086
-    printf '%s\n' ${packages_to_force_stabilize} | \
-      sort -u
-  } | \
-    grep -vxF '' | \
-    sort | \
-    uniq -u
-)
-if [ -n "${errors}" ]; then
-  # shellcheck disable=SC2086
-  >&2 printf 'Package "%s" is not in testing, staging or on the build list!\n' ${errors}
-  # shellcheck disable=SC2016
-  >&2 printf 'Did you really give it in the form "$pkgbase.$git_rev.$mod_git_rev.$repository"?\n'
-  exit 2
-fi
-
-errors=$(
-  {
-    # shellcheck disable=SC2016
-    {
-      printf 'SELECT DISTINCT CONCAT('
-      printf '`package_sources`.`%s`,".",' \
-        'pkgbase' 'git_revision' 'mod_git_revision'
-      printf '`upstream_repositories`.`name`)'
-      printf ' FROM `binary_packages`'
-      mysql_join_binary_packages_repositories
-      mysql_join_binary_packages_build_assignments
-      mysql_join_build_assignments_package_sources
-      mysql_join_package_sources_upstream_repositories
-      # consider all packages less stable than "staging"
-      printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
-      printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`'
-      printf ' WHERE `repository_stabilities`.`name` = "staging"'
-    } | \
-      mysql_run_query | \
-      sed 'p'
-    # shellcheck disable=SC2086
-    printf '%s\n' ${packages_to_force_unstage} | \
-      sort -u
-  } | \
-    grep -vxF '' | \
-    sort | \
-    uniq -u
-)
-if [ -n "${errors}" ]; then
-  # shellcheck disable=SC2086
-  >&2 printf 'Package "%s" is not in staging or on the build list!\n' ${errors}
-  # shellcheck disable=SC2016
-  >&2 printf 'Did you really give it in the form "$pkgbase.$git_rev.$mod_git_rev.$repository"?\n'
-  exit 2
-fi
-
-# Create a lock file and a trap.
-
-exec 9> "${build_list_lock_file}"
+exec 9> "${package_database_lock_file}"
 if ! flock ${block_flag} 9; then
-  >&2 echo 'come back (shortly) later - I cannot lock build list.'
-  exit 0
-fi
-
-exec 8> "${package_database_lock_file}"
-if ! flock ${block_flag} 8; then
   >&2 echo 'come back (shortly) later - I cannot lock package database.'
   exit 0
 fi
 
-exec 7> "${sanity_check_lock_file}"
-if ! flock -s ${block_flag} 7; then
+exec 8> "${sanity_check_lock_file}"
+if ! flock -s ${block_flag} 8; then
   >&2 echo 'come back (shortly) later - sanity-check currently running.'
   exit 0
 fi
 
-clean_up_lock_file() {
-  rm -f "${package_database_lock_file}" "${build_list_lock_file}"
-  rm -rf --one-file-system "${tmp_dir}"
-}
-
-trap clean_up_lock_file EXIT
-
 # shellcheck disable=SC2119
 mysql_cleanup
 
-# sanity check
-
-for ending in 'done' 'tested'; do
-  errors=$(
-    find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
-      sed 's|\(\.[^.]\+\)\{4\}$||' | \
-      sort | \
-      uniq -d
-  )
-  if [ -n "${errors}" ]; then
-    >&2 echo 'Removing duplicates not yet implemented:'
-    >&2 echo "${errors}"
-    exit 42
-  fi
-done
-
-# shellcheck disable=SC2016
-{
-  printf 'CALL calculate_maximal_moveable_set("testing");\n'
-  printf 'SELECT "mv",'
-  mysql_package_name_query
-  printf ',`repositories`.`name`,`new_repo`.`name`'
-  printf ' FROM `moveable_binary_packages`'
-  printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`'
-  mysql_join_binary_packages_repositories
-  mysql_join_binary_packages_architectures
-  mysql_join_binary_packages_build_assignments
-  mysql_join_build_assignments_package_sources
-  mysql_join_package_sources_upstream_repositories
-  mysql_join_upstream_repositories_repository_moves
-  printf ' AND `repositories`.`id`=`repository_moves`.`from_repository`'
-  printf ' JOIN `repositories` AS `new_repo` ON `new_repo`.`id`=`repository_moves`.`to_repository`'
-  printf ';\n'
-  printf 'SELECT "rm",'
-  mysql_package_name_query
-  printf ',`repositories`.`name`'
-  printf ' FROM `replaced_binary_packages`'
-  printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`'
-  mysql_join_binary_packages_repositories
-  mysql_join_binary_packages_architectures
-  printf ';\n'
-} | \
-  mysql_run_query -N --raw --batch
-
-# packages which are done
-
-find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
-  sed '
-    s|\.done$||
-  ' | \
-  sort -u > \
-  "${tmp_dir}/done-packages"
-
-# packages still on the build-list
-
-grep -vxF 'break_loops' "${work_dir}/build-list" | \
-  tr ' ' '.' | \
-  sort -u > \
-  "${tmp_dir}/build-list-packages"
-
-find "${work_dir}/package-infos" -name '*.groups' \
-  -exec grep -qxF 'base' {} \; \
-  -printf '%f\n' | \
-  sed '
-    s|\.groups$||
-  ' | \
-  sort -u > \
-  "${tmp_dir}/base-packages"
-
-{
-  # shellcheck disable=SC2086
-  printf '%s\n' ${packages_to_force_unstage}
-  print_list_of_archaic_packages 'build-list'
-} > \
-  "${tmp_dir}/force-unstage-packages"
-
->&2 printf 'calculate what packages should be unstaged ...'
-find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-unstage-packages" > \
-  "${tmp_dir}/unstage-packages"
->&2 printf ' ok.\n'
-
-# no base packages on the build list anymore?
-if [ -z "$(
-    join -j 1 \
-      "${tmp_dir}/base-packages" \
-      "${tmp_dir}/build-list-packages"
-  )" ]; then
-
-  >&2 echo 'db-update unstage: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages'
-  for s in "${tmp_dir}/all-builds" "${tmp_dir}/all-depends"; do
-    sed '/ base$/d' "${s}" > \
-      "${s}.no-base"
-  done
+for source_stability in 'testing' 'staging'; do
+  find "${tmp_dir}" -mindepth 1 -delete
 
-  >&2 printf 'calculate what packages should be unstaged ...'
-  find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-unstage-packages" > \
-    "${tmp_dir}/unstage-packages"
-  >&2 printf ' ok.\n'
+  # shellcheck disable=SC2016
+  {
+    printf 'CALL calculate_maximal_moveable_set("%s");\n' \
+      "${source_stability}"
+
+    printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
+    printf 'INSERT IGNORE INTO `rps` (`id`)'
+    printf ' SELECT `moveable_binary_packages`.`to_repository`'
+    printf ' FROM `moveable_binary_packages`;\n'
+    printf 'INSERT IGNORE INTO `rps` (`id`)'
+    printf ' SELECT `binary_packages`.`repository`'
+    printf ' FROM `moveable_binary_packages`'
+    printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`;\n'
+    printf 'INSERT IGNORE INTO `rps` (`id`)'
+    printf ' SELECT `binary_packages`.`repository`'
+    printf ' FROM `replaced_binary_packages`'
+    printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`;\n'
+
+    printf 'SELECT "repositories",`repositories`.`name`'
+    printf ' FROM `repositories`'
+    printf ' JOIN `rps` ON `rps`.`id`=`repositories`.`id`;\n'
+
+    printf 'SELECT "mv.id",`moveable_binary_packages`.`id`,`moveable_binary_packages`.`to_repository`'
+    printf ' FROM `moveable_binary_packages`;\n'
+
+    printf 'SELECT "mv",'
+    mysql_package_name_query
+    printf ',`repositories`.`name`,`new_repo`.`name`'
+    printf ' FROM `moveable_binary_packages`'
+    printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_repositories
+    mysql_join_binary_packages_architectures
+    printf ' JOIN `repositories` AS `new_repo` ON `new_repo`.`id`=`moveable_binary_packages`.`to_repository`'
+    printf ';\n'
+
+    printf 'SELECT "rm.id",`replaced_binary_packages`.`id`'
+    printf ' FROM `replaced_binary_packages`;\n'
+
+    printf 'SELECT "rm",'
+    mysql_package_name_query
+    printf ',`repositories`.`name`'
+    printf ' FROM `replaced_binary_packages`'
+    printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`'
+    mysql_join_binary_packages_repositories
+    mysql_join_binary_packages_architectures
+    printf ';\n'
+  } | \
+    mysql_run_query | \
+    tr '\t' ' ' | \
+    grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
+    while read -r what content; do
+      printf '%s\n' "${content}" >> \
+        "${tmp_dir}/${what}"
+    done
 
-fi
+  if [ ! -s "${tmp_dir}/repositories" ]; then
+    >&2 echo 'Nothing to move from %s.' "${source_stability}"
+    continue
+  fi
 
-{
   # shellcheck disable=SC2086
-  printf '%s\n' ${packages_to_force_stabilize}
-  print_list_of_archaic_packages 'build-list' 'staging' 'testing'
-} > \
-  "${tmp_dir}/force-stabilize-packages"
-
-# calculate what packages should be stabilized
-
-{
-  cat "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages"
-  find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.testing' -printf '%f\n' | \
-    sed 's|\.testing$||'
-} | \
-  sort -u > \
-  "${tmp_dir}/keep-packages"
-
-find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.tested' -printf '%f\n' | \
-    sed 's|\.tested$||' > \
-  "${tmp_dir}/stabilize-packages"
-
-# no base packages on the build list or in staging anymore?
-if [ -z "$(
-    cat \
-      "${tmp_dir}/build-list-packages" \
-      "${tmp_dir}/done-packages" | \
-      sort -u | \
-      join -j 1 \
-        "${tmp_dir}/base-packages" \
-        -
-  )" ]; then
-
-  >&2 echo 'db-update stabilize: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages'
-  >&2 printf 'calculate what packages should be stabilized ...'
-  find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-stabilize-packages" | \
-    sponge "${tmp_dir}/stabilize-packages"
-  >&2 printf ' ok.\n'
-
-else
-
-  >&2 printf 'calculate what packages should be stabilized ...'
-  find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-stabilize-packages" | \
-    sponge "${tmp_dir}/stabilize-packages"
-  >&2 printf ' ok.\n'
-
-fi
+  for s in "${tmp_dir}/"*; do
+    sort -u "${s}" | \
+      sponge "${s}"
+  done
 
-# unlock build list
+  # receive the repository databases from the master mirror
+  mkdir "${tmp_dir}/dbs"
+  while read -r repo; do
+    mkdir "${tmp_dir}/dbs/${repo}"
+    # shellcheck disable=SC2086
+    ${master_mirror_rsync_command} \
+      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
+      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
+      "${tmp_dir}/dbs/${repo}/"
+    tar -Oxzf "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz" --wildcards '*/desc' | \
+      sed -n '
+        /^%FILENAME%$/{
+          N
+          s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/
+          T
+          p
+        }
+      '
+  done < \
+    "${tmp_dir}/repositories" | \
+    while read -r old new; do
+      for file in 'rm' 'mv'; do
+        sed -i '
+          s/\(\s\|^\)'"$(str_to_regex "${old}")"'\(\s\|$\)/\1'"${new}"'\2/
+        ' "${tmp_dir}/${file}"
+      done
+    done
 
-rm -f "${build_list_lock_file}"
-flock -u 9
+  # remove to-be-deleted packages
+  while read -r pkgname repo; do
+    cd "${tmp_dir}/dbs/${repo}"
+    repo-remove -q "${repo}.db.tar.gz" "${pkgname%-*-*-*}"
+  done < \
+    "${tmp_dir}/rm"
 
-clean_up_lock_file() {
-  rm -rf --one-file-system "${tmp_dir}"
-  rm -f "${package_database_lock_file}"
-}
+  # delete and add moved packages
+  mkdir "${tmp_dir}/transit"
+  while read -r pkgname from_repo to_repo; do
+    ${master_mirror_rsync_command} \
+      "${master_mirror_rsync_directory}/i686/${from_repo}/${pkgname}" \
+      "${master_mirror_rsync_directory}/i686/${from_repo}/${pkgname}.sig" \
+      "${tmp_dir}/transit/"
+    cd "${tmp_dir}/dbs/${from_repo}"
+    repo-remove -q "${from_repo}.db.tar.gz" "${pkgname%-*-*-*}"
+    cd "${tmp_dir}/dbs/${to_repo}"
+    repo-add -q "${to_repo}.db.tar.gz" "${tmp_dir}/transit/${pkgname}"
+    rm \
+      "${tmp_dir}/transit/${pkgname}" \
+      "${tmp_dir}/transit/${pkgname}.sig"
+  done < \
+    "${tmp_dir}/mv"
 
-# testing -> stable
+  # move the packages remotely via sftp
+  {
+    sed '
+      s,^\(\S\+\) \(\S\+\)$,rm "i686/\2/\1"\nrm "i686/\2/\1.sig",
+    ' "${tmp_dir}/rm"
+    sed '
+      s,^\(\S\+\) \(\S\+\) \(\S\+\)$,rename "i686/\2/\1" "i686/\3/\1"\nrename "i686/\2/\1.sig" "i686/\3/\1.sig",
+    ' "${tmp_dir}/mv"
+    echo 'quit'
+  } | \
+    if ${no_action}; then
+      sed 's|^|sftp: |'
+    else
+      ${master_mirror_sftp_command}
+    fi
 
-while read -r package; do
-  if [ -z "${package}" ]; then
-    continue
-  fi
-  printf '%s %s %s\n' \
-    "${package}" \
-    "$(official_or_community "${package}" 'testing')" \
-    "$(repository_of_package "${package}")"
-done < \
-  "${tmp_dir}/stabilize-packages" | \
-  sponge "${tmp_dir}/stabilize-packages"
-
-# staging -> testing
-
-while read -r package; do
-  if [ -z "${package}" ]; then
+  if ${no_action}; then
     continue
   fi
-  printf '%s %s %s\n' \
-    "${package}" \
-    "$(official_or_community "${package}" 'staging')" \
-    "$(official_or_community "${package}" 'testing')"
-done < \
-  "${tmp_dir}/unstage-packages" | \
-  sponge "${tmp_dir}/unstage-packages"
-
-if [ -s "${tmp_dir}/stabilize-packages" ]; then
-  >&2 printf 'move packages from *testing/ to the stable repos ...'
-  move_packages "${tmp_dir}/stabilize-packages"
-  >&2 printf ' ok.\n'
-else
-  >&2 printf 'Nothing to move from *testing to the stable repos.\n'
-fi
 
-if [ -s "${tmp_dir}/unstage-packages" ]; then
-  >&2 printf 'move packages from *staging to *testing ...'
-  move_packages "${tmp_dir}/unstage-packages"
-  >&2 printf ' ok.\n'
-else
-  >&2 printf 'Nothing to move from *staging to *testing.\n'
-fi
+  # and push our local *.db.tar.gz via rsync
+  while read -r repo; do
+    # shellcheck disable=SC2086
+    ${master_mirror_rsync_command} \
+      "${tmp_dir}/dbs/${repo}/${repo}.db."* \
+      "${tmp_dir}/dbs/${repo}/${repo}.files."* \
+      "${master_mirror_rsync_directory}/i686/${repo}/"
+  done < \
+    "${tmp_dir}/repositories"
 
-clean_up_lock_file
+  # shellcheck disable=SC2016
+  {
+    printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+    printf 'CREATE TEMPORARY TABLE `moved_binary_packages` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`;\n' \
+      "${tmp_dir}/mv.id" 'moved_binary_packages' \
+      "${tmp_dir}/rm.id" 'replaced_binary_packages'
+    printf 'DELETE `binary_packages` FROM `binary_packages`'
+    printf ' JOIN `replaced_binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`;\n'
+    printf 'UPDATE `binary_packages`'
+    printf ' JOIN `moved_binary_packages` ON `binary_packages`.`id`=`moved_binary_packages`.`id`'
+    printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n'
+  } | \
+    mysql_run_query
+done
-- 
cgit v1.2.3-54-g00ecf


From 920f3351e045c9e2d9aa18706dc86557dd59cc5d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 09:08:51 +0100
Subject: bin/db-update: trigger_mirror_refreshs after change

---
 bin/db-update | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index a7daa7c..d79cc37 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -248,6 +248,8 @@ for source_stability in 'testing' 'staging'; do
   done < \
     "${tmp_dir}/repositories"
 
+  trigger_mirror_refreshs
+
   # shellcheck disable=SC2016
   {
     printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
-- 
cgit v1.2.3-54-g00ecf


From 02ac06c92a7fe85626819ede0072928e7ac05130 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 10:12:02 +0100
Subject: bin/db-copy new for repo-adding a package which is known from another
 repository without needing the package file

---
 bin/db-copy | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100755 bin/db-copy

diff --git a/bin/db-copy b/bin/db-copy
new file mode 100755
index 0000000..ab0201a
--- /dev/null
+++ b/bin/db-copy
@@ -0,0 +1,92 @@
+#!/bin/sh
+
+# Rudimentary copy a package from one repository to another.
+# Note, that we do _not_ need to have the package itself, since all
+# relevant information is already in the original package database.
+
+# "Rudimentary" means the following restrictions:
+#   - no arguments are accepted
+#   - no database signatures are handled
+#   - only *.db.tar.gz and *.files.tar.gz are recognized as database
+
+usage() {
+  >&2 echo 'usage:'
+  >&2 echo ' db-copy from-repo.db.tar.gz to-repo.db.tar.gz package1 package2 ...'
+  >&2 echo
+  >&2 echo 'Note, that the packages must be given with version, e.g. "linux-4.15.7-1.0".'
+  exit 2
+}
+
+if [ $# -le 2 ]; then
+  usage
+fi
+
+from_repo="$1"
+to_repo="$2"
+shift
+shift
+
+tmp_dir=$(mktemp -d)
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+# extract the databases
+for repo in 'from' 'to'; do
+  for archive in 'db' 'files'; do
+    eval 'repo_db="${'"${repo}"'_repo}"'
+    if [ "${repo_db}" = "${repo_db%.db.tar.gz}" ]; then
+      >&2 printf '"%s" has an invalid suffix.\n' "${repo_db}"
+      usage
+    fi
+    if [ "${archive}" = 'files' ]; then
+      repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+    fi
+    if [ ! -f "${repo_db}" ]; then
+      >&2 printf 'Cannot open file "%s".\n' "${repo_db}"
+      usage
+    fi
+    mkdir "${tmp_dir}/${repo}.${archive}"
+    bsdtar -C "${tmp_dir}/${repo}.${archive}" -xf "${repo_db}"
+  done
+done
+
+# move the packages
+for package in "$@"; do
+  errors=$(
+    find "${tmp_dir}/to.db" "${tmp_dir}/to.files" -mindepth 1 -maxdepth 1 \
+      -printf '%f\n' | \
+      sed 's/-[^-]\+-[^-]\+$//' | \
+      grep -xF "${package%-*-*}"
+  )
+  if [ -n "${errors}" ]; then
+    >&2 printf 'The target repository "%s" already contains the following packages - "repo-remove" them first:\n' \
+      "${to_repo}"
+    >&2 printf '%s\n' "${errors}"
+    exit 2
+  fi
+  for archive in 'db' 'files'; do
+    if [ ! -d "${tmp_dir}/from.${archive}/${package}" ]; then
+      >&2 printf 'Repository "%s" does not contain package "%s"\n' \
+        "${from_repo}" "${package}"
+      exit 2
+    fi
+    mv "${tmp_dir}/from.${archive}/${package}" "${tmp_dir}/to.${archive}/"
+  done
+done
+
+# pack the database
+for archive in 'db' 'files'; do
+  repo_db="${to_repo}"
+  if [ "${archive}" = 'files' ]; then
+    repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+  fi
+  bsdtar -C "${tmp_dir}" -czf "${tmp_dir}/${repo_db##*/}" --strip-components=1 "to.${archive}"
+done
+
+# move the database in place
+for archive in 'db' 'files'; do
+  repo_db="${to_repo}"
+  if [ "${archive}" = 'files' ]; then
+    repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+  fi
+  mv "${tmp_dir}/${repo_db##*/}" "${repo_db}"
+done
-- 
cgit v1.2.3-54-g00ecf


From b5bacd8de3c56f996f8fdd27365e735e71e0c152 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 10:28:52 +0100
Subject: bin/db-update: use repo-copy to avoid needing to pull all moving
 packages

---
 bin/db-update | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index d79cc37..a1622ec 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -202,19 +202,12 @@ for source_stability in 'testing' 'staging'; do
     "${tmp_dir}/rm"
 
   # delete and add moved packages
-  mkdir "${tmp_dir}/transit"
   while read -r pkgname from_repo to_repo; do
-    ${master_mirror_rsync_command} \
-      "${master_mirror_rsync_directory}/i686/${from_repo}/${pkgname}" \
-      "${master_mirror_rsync_directory}/i686/${from_repo}/${pkgname}.sig" \
-      "${tmp_dir}/transit/"
-    cd "${tmp_dir}/dbs/${from_repo}"
-    repo-remove -q "${from_repo}.db.tar.gz" "${pkgname%-*-*-*}"
-    cd "${tmp_dir}/dbs/${to_repo}"
-    repo-add -q "${to_repo}.db.tar.gz" "${tmp_dir}/transit/${pkgname}"
-    rm \
-      "${tmp_dir}/transit/${pkgname}" \
-      "${tmp_dir}/transit/${pkgname}.sig"
+    repo-copy \
+      "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" \
+      "${tmp_dir}/dbs/${to_repo}/${to_repo}.db.tar.gz" \
+      "${pkgname%-*}"
+    repo-remove -q "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" "${pkgname%-*-*-*}"
   done < \
     "${tmp_dir}/mv"
 
-- 
cgit v1.2.3-54-g00ecf


From 2dd83bdb652078561a80a183a4ea84162b82814c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 10:41:11 +0100
Subject: bin/db-copy -> bin/repo-copy

---
 bin/db-copy   | 92 -----------------------------------------------------------
 bin/repo-copy | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 92 deletions(-)
 delete mode 100755 bin/db-copy
 create mode 100755 bin/repo-copy

diff --git a/bin/db-copy b/bin/db-copy
deleted file mode 100755
index ab0201a..0000000
--- a/bin/db-copy
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/bin/sh
-
-# Rudimentary copy a package from one repository to another.
-# Note, that we do _not_ need to have the package itself, since all
-# relevant information is already in the original package database.
-
-# "Rudimentary" means the following restrictions:
-#   - no arguments are accepted
-#   - no database signatures are handled
-#   - only *.db.tar.gz and *.files.tar.gz are recognized as database
-
-usage() {
-  >&2 echo 'usage:'
-  >&2 echo ' db-copy from-repo.db.tar.gz to-repo.db.tar.gz package1 package2 ...'
-  >&2 echo
-  >&2 echo 'Note, that the packages must be given with version, e.g. "linux-4.15.7-1.0".'
-  exit 2
-}
-
-if [ $# -le 2 ]; then
-  usage
-fi
-
-from_repo="$1"
-to_repo="$2"
-shift
-shift
-
-tmp_dir=$(mktemp -d)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-# extract the databases
-for repo in 'from' 'to'; do
-  for archive in 'db' 'files'; do
-    eval 'repo_db="${'"${repo}"'_repo}"'
-    if [ "${repo_db}" = "${repo_db%.db.tar.gz}" ]; then
-      >&2 printf '"%s" has an invalid suffix.\n' "${repo_db}"
-      usage
-    fi
-    if [ "${archive}" = 'files' ]; then
-      repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
-    fi
-    if [ ! -f "${repo_db}" ]; then
-      >&2 printf 'Cannot open file "%s".\n' "${repo_db}"
-      usage
-    fi
-    mkdir "${tmp_dir}/${repo}.${archive}"
-    bsdtar -C "${tmp_dir}/${repo}.${archive}" -xf "${repo_db}"
-  done
-done
-
-# move the packages
-for package in "$@"; do
-  errors=$(
-    find "${tmp_dir}/to.db" "${tmp_dir}/to.files" -mindepth 1 -maxdepth 1 \
-      -printf '%f\n' | \
-      sed 's/-[^-]\+-[^-]\+$//' | \
-      grep -xF "${package%-*-*}"
-  )
-  if [ -n "${errors}" ]; then
-    >&2 printf 'The target repository "%s" already contains the following packages - "repo-remove" them first:\n' \
-      "${to_repo}"
-    >&2 printf '%s\n' "${errors}"
-    exit 2
-  fi
-  for archive in 'db' 'files'; do
-    if [ ! -d "${tmp_dir}/from.${archive}/${package}" ]; then
-      >&2 printf 'Repository "%s" does not contain package "%s"\n' \
-        "${from_repo}" "${package}"
-      exit 2
-    fi
-    mv "${tmp_dir}/from.${archive}/${package}" "${tmp_dir}/to.${archive}/"
-  done
-done
-
-# pack the database
-for archive in 'db' 'files'; do
-  repo_db="${to_repo}"
-  if [ "${archive}" = 'files' ]; then
-    repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
-  fi
-  bsdtar -C "${tmp_dir}" -czf "${tmp_dir}/${repo_db##*/}" --strip-components=1 "to.${archive}"
-done
-
-# move the database in place
-for archive in 'db' 'files'; do
-  repo_db="${to_repo}"
-  if [ "${archive}" = 'files' ]; then
-    repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
-  fi
-  mv "${tmp_dir}/${repo_db##*/}" "${repo_db}"
-done
diff --git a/bin/repo-copy b/bin/repo-copy
new file mode 100755
index 0000000..8566d7c
--- /dev/null
+++ b/bin/repo-copy
@@ -0,0 +1,92 @@
+#!/bin/sh
+
+# Rudimentary copy a package from one repository to another.
+# Note, that we do _not_ need to have the package itself, since all
+# relevant information is already in the original package database.
+
+# "Rudimentary" means the following restrictions:
+#   - no arguments are accepted
+#   - no database signatures are handled
+#   - only *.db.tar.gz and *.files.tar.gz are recognized as database
+
+usage() {
+  >&2 echo 'usage:'
+  >&2 echo ' repo-copy from-repo.db.tar.gz to-repo.db.tar.gz package1 package2 ...'
+  >&2 echo
+  >&2 echo 'Note, that the packages must be given with version, e.g. "linux-4.15.7-1.0".'
+  exit 2
+}
+
+if [ $# -le 2 ]; then
+  usage
+fi
+
+from_repo="$1"
+to_repo="$2"
+shift
+shift
+
+tmp_dir=$(mktemp -d)
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+# extract the databases
+for repo in 'from' 'to'; do
+  for archive in 'db' 'files'; do
+    eval 'repo_db="${'"${repo}"'_repo}"'
+    if [ "${repo_db}" = "${repo_db%.db.tar.gz}" ]; then
+      >&2 printf '"%s" has an invalid suffix.\n' "${repo_db}"
+      usage
+    fi
+    if [ "${archive}" = 'files' ]; then
+      repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+    fi
+    if [ ! -f "${repo_db}" ]; then
+      >&2 printf 'Cannot open file "%s".\n' "${repo_db}"
+      usage
+    fi
+    mkdir "${tmp_dir}/${repo}.${archive}"
+    bsdtar -C "${tmp_dir}/${repo}.${archive}" -xf "${repo_db}"
+  done
+done
+
+# move the packages
+for package in "$@"; do
+  errors=$(
+    find "${tmp_dir}/to.db" "${tmp_dir}/to.files" -mindepth 1 -maxdepth 1 \
+      -printf '%f\n' | \
+      sed 's/-[^-]\+-[^-]\+$//' | \
+      grep -xF "${package%-*-*}"
+  )
+  if [ -n "${errors}" ]; then
+    >&2 printf 'The target repository "%s" already contains the following packages - "repo-remove" them first:\n' \
+      "${to_repo}"
+    >&2 printf '%s\n' "${errors}"
+    exit 2
+  fi
+  for archive in 'db' 'files'; do
+    if [ ! -d "${tmp_dir}/from.${archive}/${package}" ]; then
+      >&2 printf 'Repository "%s" does not contain package "%s"\n' \
+        "${from_repo}" "${package}"
+      exit 2
+    fi
+    mv "${tmp_dir}/from.${archive}/${package}" "${tmp_dir}/to.${archive}/"
+  done
+done
+
+# pack the database
+for archive in 'db' 'files'; do
+  repo_db="${to_repo}"
+  if [ "${archive}" = 'files' ]; then
+    repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+  fi
+  bsdtar -C "${tmp_dir}" -czf "${tmp_dir}/${repo_db##*/}" --strip-components=1 "to.${archive}"
+done
+
+# move the database in place
+for archive in 'db' 'files'; do
+  repo_db="${to_repo}"
+  if [ "${archive}" = 'files' ]; then
+    repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+  fi
+  mv "${tmp_dir}/${repo_db##*/}" "${repo_db}"
+done
-- 
cgit v1.2.3-54-g00ecf


From 07de651e7222691c5eabef6ac156ab0f33fc3314 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 11:04:24 +0100
Subject: bin/db-update: repo-copy and repo-remove packages in bunches

---
 bin/db-update | 47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index a1622ec..ca52688 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -195,21 +195,38 @@ for source_stability in 'testing' 'staging'; do
     done
 
   # remove to-be-deleted packages
-  while read -r pkgname repo; do
-    cd "${tmp_dir}/dbs/${repo}"
-    repo-remove -q "${repo}.db.tar.gz" "${pkgname%-*-*-*}"
-  done < \
-    "${tmp_dir}/rm"
-
-  # delete and add moved packages
-  while read -r pkgname from_repo to_repo; do
-    repo-copy \
-      "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" \
-      "${tmp_dir}/dbs/${to_repo}/${to_repo}.db.tar.gz" \
-      "${pkgname%-*}"
-    repo-remove -q "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" "${pkgname%-*-*-*}"
-  done < \
-    "${tmp_dir}/mv"
+  # shellcheck disable=SC2094
+  cut -d' ' -f2 < \
+    "${tmp_dir}/rm" | \
+    sort -u | \
+    while read -r repo; do
+      grep " $(str_to_regex "${repo}")\$" "${tmp_dir}/rm" | \
+        sed '
+          s/\(-[^-]\+\)\{3\} \S\+$//
+        ' | \
+        xargs -r repo-remove -q "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz"
+    done
+
+  # copy and delete moved packages
+  # shellcheck disable=SC2094
+  cut -d' ' -f2,3 < \
+    "${tmp_dir}/mv" | \
+    sort -u | \
+    while read -r from_repo to_repo; do
+      grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \
+        sed '
+          s/-[^-]\+ \S\+ \S\+$//
+        ' | \
+        xargs -r repo-copy \
+          "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" \
+          "${tmp_dir}/dbs/${to_repo}/${to_repo}.db.tar.gz"
+      grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \
+        sed '
+          s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
+        ' | \
+        xargs -r repo-remove -q \
+          "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz"
+    done
 
   # move the packages remotely via sftp
   {
-- 
cgit v1.2.3-54-g00ecf


From 28099f340b36fcde5a42a13328aea6a39de44460 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 11:30:16 +0100
Subject: bin/db-update: columns are separated by space!

---
 bin/db-update | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/db-update b/bin/db-update
index ca52688..f87000a 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -264,7 +264,7 @@ for source_stability in 'testing' 'staging'; do
   {
     printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
     printf 'CREATE TEMPORARY TABLE `moved_binary_packages` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
-    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`;\n' \
+    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
       "${tmp_dir}/mv.id" 'moved_binary_packages' \
       "${tmp_dir}/rm.id" 'replaced_binary_packages'
     printf 'DELETE `binary_packages` FROM `binary_packages`'
-- 
cgit v1.2.3-54-g00ecf


From 84b99c0e495185eb423e9d8bc9418670b6ff9376 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 11:44:02 +0100
Subject: bin/db-update: keep .testing, .tested, .done (mostly) in sync

---
 bin/db-update | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index f87000a..adec0ef 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -9,10 +9,6 @@
 
 # TODO: separate locks for staging, testing (and stable)
 
-# TODO: sub_pkgrel=0 may or may not be omitted for the actual package :-/
-
-# TODO: keep .testing, .tested, .done in sync
-
 # shellcheck disable=SC2039
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
@@ -274,4 +270,25 @@ for source_stability in 'testing' 'staging'; do
     printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n'
   } | \
     mysql_run_query
+
+  # TODO: this may delete too much, but these files will be obsolete soon anyway ...
+  find "${work_dir}/package-states/" \
+    \( \
+      -name '*.done' -o \
+      -name '*.testing' -o \
+      -name '*.tested' \
+    \) \
+    -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/rm")" '{}' \; \
+    -delete
+  find "${work_dir}/package-states/" \
+    -name '*.tested' \
+    -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
+    -delete
+  find "${work_dir}/package-states/" \
+    -name '*.done' \
+    -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
+    -prinft '%p\n' | \
+    while read -r sf; do
+      mv "${sf}" "${sf%.done}.testing"
+    done
 done
-- 
cgit v1.2.3-54-g00ecf


From e6e0f25313dc77de3a6689fd95ec413749cf5309 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 11:47:23 +0100
Subject: bin/db-update: fix unimportant typo

---
 bin/db-update | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/db-update b/bin/db-update
index adec0ef..0c3e10d 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -152,7 +152,7 @@ for source_stability in 'testing' 'staging'; do
     done
 
   if [ ! -s "${tmp_dir}/repositories" ]; then
-    >&2 echo 'Nothing to move from %s.' "${source_stability}"
+    >&2 printf 'Nothing to move from %s.' "${source_stability}"
     continue
   fi
 
-- 
cgit v1.2.3-54-g00ecf


From cb64e2976dab9a4b691698f513e1906d698ddd77 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 11:49:21 +0100
Subject: bin/db-update: fix another typo :-)

---
 bin/db-update | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/db-update b/bin/db-update
index 0c3e10d..874f314 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -287,7 +287,7 @@ for source_stability in 'testing' 'staging'; do
   find "${work_dir}/package-states/" \
     -name '*.done' \
     -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
-    -prinft '%p\n' | \
+    -printf '%p\n' | \
     while read -r sf; do
       mv "${sf}" "${sf%.done}.testing"
     done
-- 
cgit v1.2.3-54-g00ecf


From 9dd30b6080c0c99c8c9fc4638d380649a7f79607 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 12:10:27 +0100
Subject: bin/db-update: fix another unimportant typo

---
 bin/db-update | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/db-update b/bin/db-update
index 874f314..fd3654c 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -152,7 +152,7 @@ for source_stability in 'testing' 'staging'; do
     done
 
   if [ ! -s "${tmp_dir}/repositories" ]; then
-    >&2 printf 'Nothing to move from %s.' "${source_stability}"
+    >&2 printf 'Nothing to move from %s.\n' "${source_stability}"
     continue
   fi
 
-- 
cgit v1.2.3-54-g00ecf


From b401b1f7c3b83c095adb01bfeded39d77f5d663f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 12:27:40 +0100
Subject: bin/db-update: touch mv* and rm* files in case they're not there yet

---
 bin/db-update | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index fd3654c..1c7ce94 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -156,6 +156,12 @@ for source_stability in 'testing' 'staging'; do
     continue
   fi
 
+  touch \
+    "${tmp_dir}/mv" \
+    "${tmp_dir}/mv.id" \
+    "${tmp_dir}/rm" \
+    "${tmp_dir}/rm.id"
+
   # shellcheck disable=SC2086
   for s in "${tmp_dir}/"*; do
     sort -u "${s}" | \
-- 
cgit v1.2.3-54-g00ecf


From 6639027c0396f5c62e12c9762859b0e39c0e6055 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 13:24:09 +0100
Subject: bin/db-update: absolute path for repo-copy

---
 bin/db-update | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/db-update b/bin/db-update
index 1c7ce94..0234608 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -219,7 +219,7 @@ for source_stability in 'testing' 'staging'; do
         sed '
           s/-[^-]\+ \S\+ \S\+$//
         ' | \
-        xargs -r repo-copy \
+        xargs -r "${base_dir}/bin/repo-copy" \
           "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" \
           "${tmp_dir}/dbs/${to_repo}/${to_repo}.db.tar.gz"
       grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \
-- 
cgit v1.2.3-54-g00ecf


From 8523496fc17c8afbcfff49b1164a265532a4d9c3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 13:45:56 +0100
Subject: web-scripts/dependencies.php: accept pkgbase via "b=..."

---
 web-scripts/dependencies.php | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/web-scripts/dependencies.php b/web-scripts/dependencies.php
index 4f12c52..ad9dbb8 100644
--- a/web-scripts/dependencies.php
+++ b/web-scripts/dependencies.php
@@ -9,6 +9,8 @@ $match = "";
 
 if (isset($_GET["a"]))
   $match .= " AND `architectures`.`name`=from_base64(\"" . base64_encode($_GET["a"]) . "\")";
+if (isset($_GET["b"]))
+  $match .= " AND `package_sources`.`pkgbase`=from_base64(\"" . base64_encode($_GET["b"]) . "\")";
 if (isset($_GET["p"]))
   $match .= " AND `binary_packages`.`pkgname`=from_base64(\"" . base64_encode($_GET["p"]) . "\")";
 if (isset($_GET["r"]))
@@ -30,6 +32,8 @@ if (! $result = $mysql -> query(
   " FROM `binary_packages`" .
   " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
   " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+  " JOIN `build_assignments` ON `binary_packages`.`build_assignment`= `build_assignments`.`id`" .
+  " JOIN `package_sources` ON `build_assignments`.`package_source`= `package_sources`.`id`" .
   $match .
   " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
   " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
@@ -45,6 +49,8 @@ if (! $result = $mysql -> query(
   " FROM `binary_packages`" .
   " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
   " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+  " JOIN `build_assignments` ON `binary_packages`.`build_assignment`= `build_assignments`.`id`" .
+  " JOIN `package_sources` ON `build_assignments`.`package_source`= `package_sources`.`id`" .
   $match .
   " JOIN `install_target_providers` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
   " JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`" .
-- 
cgit v1.2.3-54-g00ecf


From 1658d4718631b4186a405602596e344e0ab5cb05 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 14:09:51 +0100
Subject: unify web-scripts/broken-packages.php and web-scripts/build-list.php

---
 web-scripts/broken-packages.php | 194 ----------------------------------------
 web-scripts/build-list.php      |  54 +++++++++--
 2 files changed, 48 insertions(+), 200 deletions(-)
 delete mode 100644 web-scripts/broken-packages.php

diff --git a/web-scripts/broken-packages.php b/web-scripts/broken-packages.php
deleted file mode 100644
index 4dabf5c..0000000
--- a/web-scripts/broken-packages.php
+++ /dev/null
@@ -1,194 +0,0 @@
-<html>
-<head>
-<title>List of broken package builds</title>
-<link rel="stylesheet" type="text/css" href="/static/style.css">
-</head>
-<body>
-<a href="/build-logs/">build logs</a><br>
-<?php
-
-$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
-if ($mysql->connect_error) {
-  die("Connection failed: " . $mysql->connect_error);
-}
-
-$result = $mysql -> query(
-  "SELECT DISTINCT " .
-  "`build_assignments`.`id`," .
-  "`build_assignments`.`is_blocked`," .
-  "`package_sources`.`pkgbase`," .
-  "`package_sources`.`git_revision`," .
-  "`package_sources`.`mod_git_revision`," .
-  "`package_sources`.`uses_upstream`," .
-  "`package_sources`.`uses_modification`," .
-  "`upstream_repositories`.`name` AS `package_repository`," .
-  "`git_repositories`.`name` AS `git_repository`," .
-  "`architectures`.`name` AS `arch`," .
-  "EXISTS (SELECT * " .
-    "FROM `binary_packages` `broken_bin` " .
-    "JOIN `dependencies` ON `dependencies`.`dependent` = `broken_bin`.`id` " .
-    "JOIN `install_target_providers` ON `install_target_providers`.`install_target` = `dependencies`.`depending_on` " .
-    "JOIN `binary_packages` `to_be_built` ON `to_be_built`.`id` = `install_target_providers`.`package` " .
-    "JOIN `repositories` ON `to_be_built`.`repository` = `repositories`.`id` " .
-    "WHERE `broken_bin`.`build_assignment`=`build_assignments`.`id` ".
-    "AND `repositories`.`name`=\"build-list\" " .
-    "AND `to_be_built`.`build_assignment`!=`build_assignments`.`id`" .
-  ") AS `dependencies_pending`," .
-  "(SELECT count(*) " .
-    "FROM `build_dependency_loops` " .
-    "WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`" .
-  ") AS `loops` " .
-  "FROM `build_assignments` " .
-  "JOIN `architectures` ON `build_assignments`.`architecture` = `architectures`.`id` " .
-  "JOIN `package_sources` ON `build_assignments`.`package_source` = `package_sources`.`id` " .
-  "JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id` " .
-  "JOIN `git_repositories` ON `upstream_repositories`.`git_repository`=`git_repositories`.`id` " .
-  "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
-  "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
-  "WHERE (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL) AND `repositories`.`name`=\"build-list\""
-);
-if ($result -> num_rows > 0) {
-
-  $count = 0;
-
-  while($row = $result->fetch_assoc()) {
-
-    $fail_result = $mysql -> query(
-      "SELECT " .
-      "`fail_reasons`.`name`, " .
-      "`failed_builds`.`log_file` " .
-      "FROM `failed_builds` " .
-      "JOIN `build_assignments` ON `failed_builds`.`build_assignment`=`build_assignments`.`id` ".
-      "JOIN `fail_reasons` ON `failed_builds`.`reason`=`fail_reasons`.`id` ".
-      "WHERE `build_assignments`.`id`=".$row["id"]." " .
-      "ORDER BY `failed_builds`.`date`"
-    );
-
-    unset($reasons);
-    $rows[$count]["trials"] = $fail_result -> num_rows;
-    if ($rows[$count]["trials"] > 0) {
-      while($fail_row = $fail_result->fetch_assoc()) {
-        $reasons[$fail_row["name"]] = $fail_row["log_file"];
-      }
-    }
-    if (isset($reasons)) {
-      $to_print="";
-      foreach ($reasons as $reason => $last_log) {
-        if (file_exists("/srv/http/build-logs/error/".$last_log)) {
-          $to_print= $to_print .
-            ", <a href=\"/build-logs/error/" .
-            $last_log .
-            "\">" .
-            $reason .
-            "</a>";
-        } else {
-          $to_print= $to_print . ", " . $reason;
-        }
-      }
-      $rows[$count]["fail_reasons"]=substr($to_print,2);
-    } else {
-      $rows[$count]["fail_reasons"]="&nbsp;";
-    }
-
-    $rows[$count]["loops"] = $row["loops"];
-    $rows[$count]["pkgbase"] = $row["pkgbase"];
-    if ($row["dependencies_pending"]=="0")
-      $rows[$count]["pkgbase_print"] = $rows[$count]["pkgbase"];
-    else
-      $rows[$count]["pkgbase_print"] = "(" . $rows[$count]["pkgbase"] . ")";
-    if ($row["uses_upstream"]) {
-      $rows[$count]["git_revision"] =
-        "<a href=\"https://git.archlinux.org/svntogit/" .
-        $row["git_repository"] . ".git/tree/" .
-        $row["pkgbase"] . "/repos/" .
-        $row["package_repository"] . "-";
-      if ($row["arch"]=="any")
-        $rows[$count]["git_revision"] =
-          $rows[$count]["git_revision"] . "any";
-      else
-        $rows[$count]["git_revision"] =
-          $rows[$count]["git_revision"] . "x86_64";
-      $rows[$count]["git_revision"] =
-        $rows[$count]["git_revision"] . "?id=" .
-        $row["git_revision"];
-      $rows[$count]["git_revision"] =
-        $rows[$count]["git_revision"] . "\">" .
-        $row["git_revision"] . "</a>";
-    } else
-      $rows[$count]["git_revision"] = $row["git_revision"];
-    if ($row["uses_modification"])
-      $rows[$count]["mod_git_revision"] =
-        "<a href=\"https://github.com/archlinux32/packages/tree/" .
-        $row["mod_git_revision"] . "/" .
-        $row["package_repository"] . "/" .
-        $row["pkgbase"] . "\">" .
-        $row["mod_git_revision"] . "</a>";
-    else
-      $rows[$count]["mod_git_revision"] = $row["mod_git_revision"];
-    $rows[$count]["package_repository"] = $row["package_repository"];
-    if ($row["is_blocked"]=="") {
-      $rows[$count]["is_blocked"]="&nbsp;";
-    }
-    else {
-      $rows[$count]["is_blocked"] = preg_replace(
-        array (
-          "/FS32#(\\d+)/",
-          "/FS#(\\d+)/"
-        ),
-        array (
-          "<a href=\"https://bugs.archlinux32.org/index.php?do=details&task_id=$1\">$0</a>",
-          "<a href=\"https://bugs.archlinux.org/task/$1\">$0</a>"
-        ),
-        $row["is_blocked"]
-      );
-    }
-    $count++;
-  }
-
-  usort(
-    $rows,
-    function (array $a, array $b) {
-      if ($a["trials"] < $b["trials"])
-        return -1;
-      if ($a["trials"] > $b["trials"])
-        return 1;
-      return strcmp($a["pkgbase"],$b["pkgbase"]);
-    }
-  );
-
-  print "<table>\n";
-  print "<tr>";
-  print "<th>package</th>";
-  print "<th>git revision</th>";
-  print "<th>modification git revision</th>";
-  print "<th>package repository</th>";
-  print "<th>compilations</th>";
-  print "<th>loops</th>";
-//  print "<th>dependent</th>";
-  print "<th>build error</th>";
-  print "<th>blocked</th>";
-  print "</tr>\n";
-
-  foreach($rows as $row) {
-
-    print "<tr>";
-
-    print "<td><a href=\"/graphs/".$row["pkgbase"].".png\">".$row["pkgbase_print"]."</a></td>";
-    print "<td><p style=\"font-size:8px\">".$row["git_revision"]."</p></td>";
-    print "<td><p style=\"font-size:8px\">".$row["mod_git_revision"]."</p></td>";
-    print "<td>".$row["package_repository"]."</td>";
-    print "<td>".$row["trials"]."</td>";
-    print "<td>".$row["loops"]."</td>";
-//    <td>0</td>
-    print "<td>".$row["fail_reasons"]."</td>";
-    print "<td>".$row["is_blocked"]."</td>";
-
-    print "</tr>\n";
-  }
-
-  print "</table>\n";
-}
-
-?>
-</body>
-</html>
diff --git a/web-scripts/build-list.php b/web-scripts/build-list.php
index 1b5c359..0ca36b8 100644
--- a/web-scripts/build-list.php
+++ b/web-scripts/build-list.php
@@ -1,6 +1,11 @@
 <html>
 <head>
-<title>List of scheduled package builds</title>
+<?php
+  if (isset($_GET["all"]))
+    print "<title>List of scheduled package builds</title>\n";
+  else
+    print "<title>List of broken package builds</title>\n";
+?>
 <link rel="stylesheet" type="text/css" href="/static/style.css">
 </head>
 <body>
@@ -12,6 +17,11 @@ if ($mysql->connect_error) {
   die("Connection failed: " . $mysql->connect_error);
 }
 
+if (isset($_GET["all"]))
+  $match_broken = "";
+else
+  $match_broken = "AND (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL)";
+
 $result = $mysql -> query(
   "SELECT DISTINCT " .
   "`build_assignments`.`id`," .
@@ -45,7 +55,7 @@ $result = $mysql -> query(
   "JOIN `git_repositories` ON `upstream_repositories`.`git_repository`=`git_repositories`.`id` " .
   "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
   "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
-  "WHERE `repositories`.`name`=\"build-list\""
+  "WHERE `repositories`.`name`=\"build-list\"" . $match_broken
 );
 if ($result -> num_rows > 0) {
 
@@ -55,12 +65,39 @@ if ($result -> num_rows > 0) {
 
     $fail_result = $mysql -> query(
       "SELECT " .
-      "`failed_builds`.`id` " .
+      "`fail_reasons`.`name`, " .
+      "`failed_builds`.`log_file` " .
       "FROM `failed_builds` " .
-      "WHERE `failed_builds`.`build_assignment`=".$row["id"]
+      "JOIN `fail_reasons` ON `failed_builds`.`reason`=`fail_reasons`.`id` " .
+      "WHERE `failed_builds`.`build_assignment`=".$row["id"]." " .
+      "ORDER BY `failed_builds`.`date`"
     );
 
+    unset($reasons);
     $rows[$count]["trials"] = $fail_result -> num_rows;
+    if ($rows[$count]["trials"] > 0) {
+      while($fail_row = $fail_result->fetch_assoc()) {
+        $reasons[$fail_row["name"]] = $fail_row["log_file"];
+      }
+    }
+    if (isset($reasons)) {
+      $to_print="";
+      foreach ($reasons as $reason => $last_log) {
+        if (file_exists("/srv/http/build-logs/error/".$last_log)) {
+          $to_print= $to_print .
+            ", <a href=\"/build-logs/error/" .
+            $last_log .
+            "\">" .
+            $reason .
+            "</a>";
+        } else {
+          $to_print= $to_print . ", " . $reason;
+        }
+      }
+      $rows[$count]["fail_reasons"]=substr($to_print,2);
+    } else {
+      $rows[$count]["fail_reasons"]="&nbsp;";
+    }
 
     $rows[$count]["loops"] = $row["loops"];
     $rows[$count]["pkgbase"] = $row["pkgbase"];
@@ -81,7 +118,10 @@ if ($result -> num_rows > 0) {
         $rows[$count]["git_revision"] =
           $rows[$count]["git_revision"] . "x86_64";
       $rows[$count]["git_revision"] =
-        $rows[$count]["git_revision"] . "/\">" .
+        $rows[$count]["git_revision"] . "?id=" .
+        $row["git_revision"];
+      $rows[$count]["git_revision"] =
+        $rows[$count]["git_revision"] . "\">" .
         $row["git_revision"] . "</a>";
     } else
       $rows[$count]["git_revision"] = $row["git_revision"];
@@ -133,6 +173,7 @@ if ($result -> num_rows > 0) {
   print "<th>package repository</th>";
   print "<th>compilations</th>";
   print "<th>loops</th>";
+  print "<th>build error</th>";
   print "<th>blocked</th>";
   print "</tr>\n";
 
@@ -140,12 +181,13 @@ if ($result -> num_rows > 0) {
 
     print "<tr>";
 
-    print "<td><a href=\"/graphs/".$row["pkgbase"].".png\">".$row["pkgbase_print"]."</a></td>";
+    print "<td><a href=\"/scripts/dependencies.php?b=".$row["pkgbase"]."&r=build-list\">".$row["pkgbase_print"]."</a></td>";
     print "<td><p style=\"font-size:8px\">".$row["git_revision"]."</p></td>";
     print "<td><p style=\"font-size:8px\">".$row["mod_git_revision"]."</p></td>";
     print "<td>".$row["package_repository"]."</td>";
     print "<td>".$row["trials"]."</td>";
     print "<td>".$row["loops"]."</td>";
+    print "<td>".$row["fail_reasons"]."</td>";
     print "<td>".$row["is_blocked"]."</td>";
 
     print "</tr>\n";
-- 
cgit v1.2.3-54-g00ecf


From 8a298d36658f1f34a3107c40dec9373edbf7171c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 14:13:22 +0100
Subject: web-scripts/build-list.php: "all" <-> "broken"

---
 web-scripts/build-list.php | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/web-scripts/build-list.php b/web-scripts/build-list.php
index 0ca36b8..1bb985d 100644
--- a/web-scripts/build-list.php
+++ b/web-scripts/build-list.php
@@ -1,10 +1,10 @@
 <html>
 <head>
 <?php
-  if (isset($_GET["all"]))
-    print "<title>List of scheduled package builds</title>\n";
-  else
+  if (isset($_GET["broken"]))
     print "<title>List of broken package builds</title>\n";
+  else
+    print "<title>List of scheduled package builds</title>\n";
 ?>
 <link rel="stylesheet" type="text/css" href="/static/style.css">
 </head>
@@ -17,10 +17,10 @@ if ($mysql->connect_error) {
   die("Connection failed: " . $mysql->connect_error);
 }
 
-if (isset($_GET["all"]))
-  $match_broken = "";
-else
+if (isset($_GET["broken"]))
   $match_broken = "AND (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL)";
+else
+  $match_broken = "";
 
 $result = $mysql -> query(
   "SELECT DISTINCT " .
-- 
cgit v1.2.3-54-g00ecf


From 2acef10964ec3f7027981f45a072fd5e4aab3722 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 15:47:43 +0100
Subject: bin/db-update: should not run mysql_cleanup - it does not have the
 necessary locks!

---
 bin/db-update | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 0234608..27cb1d6 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -90,9 +90,6 @@ if ! flock -s ${block_flag} 8; then
   exit 0
 fi
 
-# shellcheck disable=SC2119
-mysql_cleanup
-
 for source_stability in 'testing' 'staging'; do
   find "${tmp_dir}" -mindepth 1 -delete
 
-- 
cgit v1.2.3-54-g00ecf


From 9ede9a11566506384d5ce13736a9153313a14535 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 16:09:53 +0100
Subject: bin/get-package-updates: do not remove to-be-decided packages (of
 other running instance!) if we cannot acquire a lock

---
 bin/get-package-updates | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index ffcb078..5285646 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -16,9 +16,6 @@
 
 # TODO: keep database clean in case of abort
 
-# TODO: It seems, updating present build assignments in the database is broken
-# ... they seem to get deleted
-
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
@@ -136,22 +133,6 @@ delete_package() {
   fi
 }
 
-# create tmp_dir and trap
-clean_up() {
-  rm -rf --one-file-system "${tmp_dir}"
-  # shellcheck disable=SC2016
-  {
-    printf 'DELETE FROM `binary_packages`'
-    printf ' WHERE `binary_packages`.`repository`=('
-      printf 'SELECT `repositories`.`id` FROM `repositories`'
-      printf ' WHERE `repositories`.`name`="to-be-decided"'
-    printf ');\n'
-  } | \
-    mysql_run_query
-}
-tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)
-trap 'clean_up' EXIT
-
 # Update git repositories (official packages, community packages and the repository of package customizations).
 
 for repo in ${repo_names}; do
@@ -202,6 +183,8 @@ if ! flock -s ${block_flag} 8; then
   exit
 fi
 
+trap mysql_cleanup EXIT
+
 # shellcheck disable=SC2119
 mysql_cleanup
 
-- 
cgit v1.2.3-54-g00ecf


From fe3ae1c23523c824fb3fe1e7904c24b949b7ce6f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 16:14:37 +0100
Subject: bin/get-package-updates: clarify comment

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 5285646..0276b7e 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# check for packages that need to be built, and build a list in the proper build order
+# check for packages that need to be built
 # Details:
 #  https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates
 
-- 
cgit v1.2.3-54-g00ecf


From 08f69a1eb1f6a71751cda4df67213fa086eeae53 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 20:32:43 +0100
Subject: web-scripts/build-list.php: "?show=next" new

---
 web-scripts/build-list.php | 42 ++++++++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/web-scripts/build-list.php b/web-scripts/build-list.php
index 1bb985d..98217de 100644
--- a/web-scripts/build-list.php
+++ b/web-scripts/build-list.php
@@ -1,27 +1,32 @@
 <html>
 <head>
 <?php
-  if (isset($_GET["broken"]))
-    print "<title>List of broken package builds</title>\n";
-  else
-    print "<title>List of scheduled package builds</title>\n";
-?>
-<link rel="stylesheet" type="text/css" href="/static/style.css">
-</head>
-<body>
-<a href="/build-logs/">build logs</a><br>
-<?php
+
+if (isset($_GET["show"]))
+  $to_show=$_GET["show"];
+else
+  $to_show="all";
+
+if ($to_show == "all")
+  $match = "";
+elseif ($to_show == "broken")
+  $match = " AND (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL)";
+elseif ($to_show == "next")
+  $match = "";
+else
+  die();
+
+print "<title>List of " . $to_show . " package builds</title>\n";
+print "<link rel=\"stylesheet\" type=\"text/css\" href=\"/static/style.css\">\n";
+print "</head>\n";
+print "<body>\n";
+print "<a href=\"/build-logs/\">build logs</a><br>\n";
 
 $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
 if ($mysql->connect_error) {
   die("Connection failed: " . $mysql->connect_error);
 }
 
-if (isset($_GET["broken"]))
-  $match_broken = "AND (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL)";
-else
-  $match_broken = "";
-
 $result = $mysql -> query(
   "SELECT DISTINCT " .
   "`build_assignments`.`id`," .
@@ -55,7 +60,7 @@ $result = $mysql -> query(
   "JOIN `git_repositories` ON `upstream_repositories`.`git_repository`=`git_repositories`.`id` " .
   "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
   "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
-  "WHERE `repositories`.`name`=\"build-list\"" . $match_broken
+  "WHERE `repositories`.`name`=\"build-list\"" . $match
 );
 if ($result -> num_rows > 0) {
 
@@ -63,6 +68,11 @@ if ($result -> num_rows > 0) {
 
   while($row = $result->fetch_assoc()) {
 
+    if (($to_show == "next") &&
+      ($row["loops"]==0) &&
+      ($row["dependencies_pending"]==1))
+      continue;
+
     $fail_result = $mysql -> query(
       "SELECT " .
       "`fail_reasons`.`name`, " .
-- 
cgit v1.2.3-54-g00ecf


From 83effa9743fda2a0ca064a1dc0fc4d515088a858 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 9 Mar 2018 22:56:29 +0100
Subject: bin/bootstrap-mysql: add TODO

---
 bin/bootstrap-mysql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index afe1850..5f938fc 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -310,6 +310,8 @@ if [ ! "$1" = 'slim' ]; then
     # Every package which is replaced, must have its provided install_targets:
     #   a) provided by another moved or not-replaced package or
     #   b) not required by any not-replaced package.
+# TODO: additionally, every package being moved needs to have all dependencies
+# installable in the target repository !
     printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n'
     printf 'DELIMITER //\n'
     printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n'
-- 
cgit v1.2.3-54-g00ecf


From 5807cfc618e5f92b4323110546885ba267d09a38 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 10 Mar 2018 13:45:43 +0100
Subject: bin/bootstrap-mysql: calculate_maximal_moveable_set: packages are not
 moveable if they cannot be installed in the target repository

---
 bin/bootstrap-mysql | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 53 insertions(+), 3 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 5f938fc..b7e08ba 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -307,15 +307,20 @@ if [ ! "$1" = 'slim' ]; then
     # stores results in `moveable_binary_packages` and `replaced_binary_packages`
     # Give a maximal list of packages to be moved, while implementing the
     # condition from db-update:
+
     # Every package which is replaced, must have its provided install_targets:
     #   a) provided by another moved or not-replaced package or
     #   b) not required by any not-replaced package.
-# TODO: additionally, every package being moved needs to have all dependencies
-# installable in the target repository !
+
+    # Every package being moved needs to have all dependencies
+    # installable in the target repository.
+
     printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n'
     printf 'DELIMITER //\n'
     printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n'
     printf 'BEGIN\n'
+    # variables to store count of changed rows
+    printf 'DECLARE row_count_saved INT DEFAULT 0;\n'
     for copy in '' '_copy' '_copy2'; do
       printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \
         'moveable' "${copy}" \
@@ -369,6 +374,7 @@ if [ ! "$1" = 'slim' ]; then
     # now we delete all unreplaceable and unmoveable packages from the respective
     # list until no further changes are required
     printf 'REPEAT\n'
+    # create copies of our temporary tables *yuck*
     for table in 'replaced' 'moveable'; do
       for copy in '' '2'; do
         if [ "${table}" = 'moveable' ] && \
@@ -381,6 +387,50 @@ if [ ! "$1" = 'slim' ]; then
         printf ' FROM `%s_binary_packages`;\n' "${table}"
       done
     done
+    # a package is not moveable if its dependencies are not provided ...
+    printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages`'
+    printf ' FROM `replaced_binary_packages`'
+    printf ' RIGHT JOIN `moveable_binary_packages`'
+    printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`'
+    printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+    mysql_join_binary_packages_repositories
+    mysql_join_binary_packages_dependencies
+    mysql_join_dependencies_dependency_types
+    printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+    # ... by a not-deleted, "more stable" package already in place or ...
+    printf ' WHERE NOT EXISTS ('
+      printf 'SELECT * FROM `install_target_providers`'
+      mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+      mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+      printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`less_stable`'
+      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+      printf ' AND `repositories`.`stability`=`repository_stability_relations`.`more_stable`'
+      printf ' AND NOT EXISTS ('
+        printf 'SELECT * FROM `replaced_binary_packages_copy`'
+        printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`'
+      printf ')'
+    # ... by a moved package
+    printf ') AND NOT EXISTS ('
+      printf 'SELECT * FROM `install_target_providers`'
+      printf ' JOIN `moveable_binary_packages_copy` ON `moveable_binary_packages_copy`.`id`=`install_target_providers`.`package`'
+      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+    printf ');\n'
+    printf 'SET @row_count_saved = ROW_COUNT();\n'
+
+    # refresh copies of our temporary tables *yuck*
+    for table in 'replaced' 'moveable'; do
+      for copy in '' '2'; do
+        if [ "${table}" = 'moveable' ] && \
+          [ "${copy}" = '2' ]; then
+          continue
+        fi
+        printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}"
+        printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}"
+        printf ' SELECT `%s_binary_packages`.*' "${table}"
+        printf ' FROM `%s_binary_packages`;\n' "${table}"
+      done
+    done
+
     # packages which should not be replaced: ones providing something that is:
     # a) still needed by a not-replaced package x "less stable" than the target repository and
     # b) not provided by:
@@ -445,7 +495,7 @@ if [ ! "$1" = 'slim' ]; then
       printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`'
       # b) 2) ^
     printf ');\n'
-    printf 'UNTIL ROW_COUNT()=0\n'
+    printf 'UNTIL row_count_saved=0 AND ROW_COUNT()=0\n'
     printf 'END REPEAT;\n'
     for table in 'moveable' 'replaced'; do
       for copy in '' '2'; do
-- 
cgit v1.2.3-54-g00ecf


From 9714f8d729a53b4b55c72b7d6be3ba35148cdc6f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 10 Mar 2018 14:28:43 +0100
Subject: web-scripts/dependencies.php: colorize dependencies

---
 web-scripts/dependencies.php | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/web-scripts/dependencies.php b/web-scripts/dependencies.php
index ad9dbb8..58d3e39 100644
--- a/web-scripts/dependencies.php
+++ b/web-scripts/dependencies.php
@@ -18,6 +18,14 @@ if (isset($_GET["r"]))
 
 $ignore_install_targets = " AND NOT `install_targets`.`name` IN (\"base\",\"base-devel\")";
 
+$colors["stable"]="#000000";
+$colors["testing"]="#008000";
+$colors["staging"]="#00ff00";
+$colors["standalone"]="#000000";
+$colors["unbuilt"]="#ff0000";
+$colors["forbidden"]="#808080";
+$colors["virtual"]="#800080";
+
 if (! $result = $mysql -> query(
   "CREATE TEMPORARY TABLE `cons` (" .
     "`dep` BIGINT, " .
@@ -94,7 +102,7 @@ if (! $result = $mysql -> query(
 
 if ($result -> num_rows > 0)
   while ($row = $result->fetch_assoc())
-    $knots .= "\"i" . $row["id"] . "\" [label = \"" . $row["name"] . "\", fontcolor=\"#000080\"];\n";
+    $knots .= "\"i" . $row["id"] . "\" [label = \"" . $row["name"] . "\", fontcolor = \"#000080\"];\n";
 
 $pkgfile_query =
   "CONCAT(".
@@ -108,32 +116,40 @@ $pkgfile_query =
   ") AS `filename`";
 
 if (! $result = $mysql -> query(
-  "SELECT DISTINCT `binary_packages`.`id`," . $pkgfile_query .
+  "SELECT DISTINCT " .
+  "`binary_packages`.`id`," .
+  "`repository_stabilities`.`name` AS `stability`," .
+  $pkgfile_query .
   " FROM `cons`" .
   " JOIN `dependencies` ON `cons`.`dep`=`dependencies`.`id`" .
   " JOIN `binary_packages` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
   " JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`" .
-  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`"
+  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`" .
+  " JOIN `repository_stabilities` ON `repository_stabilities`.`id`=`repositories`.`stability`"
   ))
   die($mysql->error);
 
 if ($result -> num_rows > 0)
   while ($row = $result->fetch_assoc())
-    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\"];\n";
+    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\", fontcolor = \"" . $colors[$row["stability"]] . "\"];\n";
 
 if (! $result = $mysql -> query(
-  "SELECT DISTINCT `binary_packages`.`id`," . $pkgfile_query .
+  "SELECT DISTINCT " .
+  "`binary_packages`.`id`," .
+  "`repository_stabilities`.`name` AS `stability`," .
+  $pkgfile_query .
   " FROM `cons`" .
   " JOIN `install_target_providers` ON `cons`.`itp`=`install_target_providers`.`id`" .
   " JOIN `binary_packages` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
   " JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`" .
-  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`"
+  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`" .
+  " JOIN `repository_stabilities` ON `repository_stabilities`.`id`=`repositories`.`stability`"
   ))
   die($mysql->error);
 
 if ($result -> num_rows > 0)
   while ($row = $result->fetch_assoc())
-    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\"];\n";
+    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\", fontcolor = \"" . $colors[$row["stability"]] . "\"];\n";
 
 $knots = str_replace("\$","\\\$",$knots);
 $edges = str_replace("\$","\\\$",$edges);
-- 
cgit v1.2.3-54-g00ecf


From 30e05f52359521ccc4fd387e14c4147a413a2049 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 10 Mar 2018 14:30:04 +0100
Subject: web-scripts/build-list.php: add link to main page

---
 web-scripts/build-list.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/web-scripts/build-list.php b/web-scripts/build-list.php
index 98217de..cd197eb 100644
--- a/web-scripts/build-list.php
+++ b/web-scripts/build-list.php
@@ -20,6 +20,7 @@ print "<title>List of " . $to_show . " package builds</title>\n";
 print "<link rel=\"stylesheet\" type=\"text/css\" href=\"/static/style.css\">\n";
 print "</head>\n";
 print "<body>\n";
+print "<a href=\"main page\">/</a>\n";
 print "<a href=\"/build-logs/\">build logs</a><br>\n";
 
 $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
-- 
cgit v1.2.3-54-g00ecf


From e178be28f4219a3a65e3c1c41d882f5eeb400219 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 10 Mar 2018 14:31:19 +0100
Subject: web-scripts/build-list.php: link to start page should be the other
 way round :-/

---
 web-scripts/build-list.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web-scripts/build-list.php b/web-scripts/build-list.php
index cd197eb..b94970b 100644
--- a/web-scripts/build-list.php
+++ b/web-scripts/build-list.php
@@ -20,7 +20,7 @@ print "<title>List of " . $to_show . " package builds</title>\n";
 print "<link rel=\"stylesheet\" type=\"text/css\" href=\"/static/style.css\">\n";
 print "</head>\n";
 print "<body>\n";
-print "<a href=\"main page\">/</a>\n";
+print "<a href=\"/\">Start page</a>\n";
 print "<a href=\"/build-logs/\">build logs</a><br>\n";
 
 $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
-- 
cgit v1.2.3-54-g00ecf


From 91353d076cb6f770980d1fee9faa7856fde5be32 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 10 Mar 2018 14:59:36 +0100
Subject: web-scripts/dependencies.php: restrict to relevant dependencies

---
 web-scripts/dependencies.php | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/web-scripts/dependencies.php b/web-scripts/dependencies.php
index 58d3e39..3a38ac8 100644
--- a/web-scripts/dependencies.php
+++ b/web-scripts/dependencies.php
@@ -26,6 +26,8 @@ $colors["unbuilt"]="#ff0000";
 $colors["forbidden"]="#808080";
 $colors["virtual"]="#800080";
 
+$limit=200;
+
 if (! $result = $mysql -> query(
   "CREATE TEMPORARY TABLE `cons` (" .
     "`dep` BIGINT, " .
@@ -39,15 +41,18 @@ if (! $result = $mysql -> query(
   " SELECT `dependencies`.`id`,`install_target_providers`.`id`".
   " FROM `binary_packages`" .
   " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+  " JOIN `repository_stabilities` ON `repositories`.`stability`=`repository_stabilities`.`id`" .
   " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
   " JOIN `build_assignments` ON `binary_packages`.`build_assignment`= `build_assignments`.`id`" .
   " JOIN `package_sources` ON `build_assignments`.`package_source`= `package_sources`.`id`" .
   $match .
   " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+  " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
   " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
   $ignore_install_targets .
   " JOIN `install_target_providers` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`" .
-  " LIMIT 50"
+  " WHERE (`dependency_types`.`relevant_for_binary_packages` OR `repository_stabilities`.`name`=\"unbuilt\")" .
+  " LIMIT " . $limit
   ))
   die($mysql->error);
 
@@ -62,7 +67,12 @@ if (! $result = $mysql -> query(
   $match .
   " JOIN `install_target_providers` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
   " JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`" .
-  " LIMIT 50"
+  " JOIN `binary_packages` AS `d_bp` ON `dependencies`.`dependent`=`d_bp`.`id`" .
+  " JOIN `repositories` AS `d_r` ON `d_bp`.`repository`=`d_r`.`id`" .
+  " JOIN `repository_stabilities` AS `d_rs` ON `d_r`.`stability`=`d_rs`.`id`" .
+  " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
+  " WHERE (`dependency_types`.`relevant_for_binary_packages` OR `d_rs`.`name`=\"unbuilt\")" .
+  " LIMIT " . $limit
   ))
   die($mysql->error);
 
-- 
cgit v1.2.3-54-g00ecf


From e56544fe3896e9be5706e82869e5e42c80f35f9f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 10 Mar 2018 20:51:02 +0100
Subject: bin/db-update: add TODO

---
 bin/db-update | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index 27cb1d6..001f030 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -9,6 +9,9 @@
 
 # TODO: separate locks for staging, testing (and stable)
 
+# TODO: we should delete more packages than just the ones in repositories
+# where we move to (think of [extra] -> [community])
+
 # shellcheck disable=SC2039
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
-- 
cgit v1.2.3-54-g00ecf


From 6bcf73307e3bad6488a2e36dbaeaea594df475c0 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 08:57:23 +0100
Subject: lib/mysql-functions: remove unnecessary functions

---
 lib/mysql-functions | 62 -----------------------------------------------------
 1 file changed, 62 deletions(-)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index d87b79e..7a377d5 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -142,50 +142,6 @@ mysql_add_package_source() {
     mysql_run_query
 }
 
-# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository
-
-# shellcheck disable=SC2031,SC2086,SC2154
-mysql_add_binary_package() {
-  local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository'
-  local name
-  for name in ${names}; do
-    eval 'local '"${name}"
-    eval "${name}"'=$(
-      printf "%s" "$1" |
-        base64 -w0
-    )'
-    shift
-  done
-
-  {
-    printf 'INSERT IGNORE INTO binary_packages'
-    printf ' ('
-    printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted'
-    printf ') SELECT'
-    printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}"
-    printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures'
-    printf ' 0, 0, 0'
-    printf ' FROM'
-    printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures'
-    printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
-    printf ' WHERE'
-    printf ' `%s`.`name` = from_base64("%s") AND' \
-      'repositories' "${repository}" \
-      'architectures' "${architecture}"
-    printf ' `package_sources`.`%s` = from_base64("%s") AND' \
-      'pkgbase' "${pkgbase}" \
-      'git_revision' "${git_revision}" \
-      'mod_git_revision' "${mod_git_revision}"
-    printf ' `upstream_repositories`.`name` = from_base64("%s")' \
-      "${upstream_package_repository}"
-  } | \
-    sed '
-      s|, )|)|g
-      s|, FROM| FROM|g
-    ' | \
-    mysql_run_query
-}
-
 # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel
 
 # shellcheck disable=SC2031,SC2086,SC2154
@@ -235,24 +191,6 @@ mysql_show_binary_package() {
     mysql_run_query --html --column-names
 }
 
-# mysql_add_install_target $install_target
-
-# shellcheck disable=2086
-mysql_add_install_target() {
-  local install_target
-  install_target=$(
-    printf "%s" "$1" | \
-      base64 -w0
-  )
-
-  {
-    printf 'INSERT IGNORE INTO `install_targets` (`name`)'
-    printf ' VALUES (from_base64("%s"))' \
-      "${install_target}"
-  } | \
-    mysql_run_query
-}
-
 # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository
 # or
 # mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository
-- 
cgit v1.2.3-54-g00ecf


From b8f92b3a98ef521d15140fa4c28327e85f73d71e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 09:02:07 +0100
Subject: lib/mysql-functions: small improvement

---
 lib/mysql-functions | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index 7a377d5..aace33a 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -109,6 +109,9 @@ mysql_add_package_source() {
   fi
   uses_upstream=0
   for repo in ${repo_names}; do
+    if [ "${repo}" = 'archlinux32' ]; then
+      continue
+    fi
     if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
       tar -t 2>/dev/null | \
       sed 's,-x86_64/,-any/,' | \
-- 
cgit v1.2.3-54-g00ecf


From ecc8af9665697dd8b2b074cab4fc14ea780c5915 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 09:45:15 +0100
Subject: bin/db-update: only delete/move parts of state files to avoid
 unnecessary insanities

---
 bin/db-update | 47 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 001f030..01a38ad 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -277,24 +277,59 @@ for source_stability in 'testing' 'staging'; do
   } | \
     mysql_run_query
 
-  # TODO: this may delete too much, but these files will be obsolete soon anyway ...
+  sed_apply_rm=$(
+    while read -r pkg _ _; do
+      printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
+    done < \
+      "${tmp_dir}/rm"
+  )
+
+  sed_apply_remove_part_of_mv=$(
+    while read -r pkg _ _; do
+      printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
+    done < \
+      "${tmp_dir}/mv"
+  )
+
+  sed_apply_move_part_of_mv=$(
+    while read -r pkg _ _; do
+      printf '/^%s$/{\n' "$(str_to_regex "${pkg}")"
+      printf 'w %%s\n'
+      printf 'd\n'
+      printf '}\n'
+    done < \
+      "${tmp_dir}/mv"
+  )
+
   find "${work_dir}/package-states/" \
     \( \
       -name '*.done' -o \
       -name '*.testing' -o \
       -name '*.tested' \
     \) \
-    -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/rm")" '{}' \; \
-    -delete
+    -exec sed -i "${sed_apply_rm}" '{}' \;
   find "${work_dir}/package-states/" \
     -name '*.tested' \
-    -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
-    -delete
+    -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \;
   find "${work_dir}/package-states/" \
     -name '*.done' \
     -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
     -printf '%p\n' | \
     while read -r sf; do
-      mv "${sf}" "${sf%.done}.testing"
+      sed -i "$(
+        printf '%s\n' "${sed_apply_move_part_of_mv}" | \
+          sed '
+            s,%s,'"${sf%.done}.testing"',g
+          '
+      )" "${sf}"
     done
+  find "${work_dir}/package-states/" \
+    \( \
+      -name '*.done' -o \
+      -name '*.testing' -o \
+      -name '*.tested' \
+    \) \
+    -type f \
+    -not -exec test -s '{}' \; \
+    -delete
 done
-- 
cgit v1.2.3-54-g00ecf


From 4ad43a2e5edb2625291e8b0da893ddad42bb2830 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 10:52:54 +0100
Subject: web-scripts/mysql-issues.php new

---
 web-scripts/mysql-issues.php | 104 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100755 web-scripts/mysql-issues.php

diff --git a/web-scripts/mysql-issues.php b/web-scripts/mysql-issues.php
new file mode 100755
index 0000000..56d4c27
--- /dev/null
+++ b/web-scripts/mysql-issues.php
@@ -0,0 +1,104 @@
+<html>
+<head>
+<title>More and less critical issues with the database</title>
+<link rel="stylesheet" type="text/css" href="/static/style.css">
+</head>
+<body>
+<a href="/">Start page</a><br>
+<?php
+
+  $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+  if ($mysql->connect_error) {
+    die("Connection failed: " . $mysql->connect_error);
+  }
+
+  if (! $result = $mysql -> query(
+    "SELECT CONCAT(" .
+    "`repositories`.`name`,\"/\"," .
+    "`binary_packages`.`pkgname`,\"-\"," .
+    "IF(`binary_packages`.`epoch`=0,\"\",CONCAT(`binary_packages`.`epoch`,\":\"))," .
+    "`binary_packages`.`pkgver`,\"-\"," .
+    "`binary_packages`.`pkgrel`,\".\"," .
+    "`binary_packages`.`sub_pkgrel`,\"-\"," .
+    "`architectures`.`name`) AS `pkgfile`," .
+    "`install_targets`.`name` AS `install_target`," .
+    "IF(`binary_packages`.`is_to_be_deleted`,1,0) AS `is_to_be_deleted`" .
+    " FROM `binary_packages`" .
+    " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+    " AND `repositories`.`is_on_master_mirror`" .
+    " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+    " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
+    " AND `dependency_types`.`relevant_for_binary_packages`" .
+    " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
+    " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+    " WHERE NOT EXISTS (" .
+      "SELECT * FROM `install_target_providers`" .
+      " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
+    ") ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
+  ))
+    die($mysql -> error);
+
+  if ($result -> num_rows > 0) {
+
+    while ($row = $result->fetch_assoc()) {
+      if ($row["is_to_be_deleted"]==1)
+        print "<font color=\"#00ff00\">(marked as to-be-deleted) ";
+      else
+        print "<font color=\"#ff0000\">";
+      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package.<br>\n";
+      print "</font>";
+    }
+
+  }
+
+  if (! $result = $mysql -> query(
+    "SELECT CONCAT(" .
+    "`repositories`.`name`,\"/\"," .
+    "`binary_packages`.`pkgname`,\"-\"," .
+    "IF(`binary_packages`.`epoch`=0,\"\",CONCAT(`binary_packages`.`epoch`,\":\"))," .
+    "`binary_packages`.`pkgver`,\"-\"," .
+    "`binary_packages`.`pkgrel`,\".\"," .
+    "`binary_packages`.`sub_pkgrel`,\"-\"," .
+    "`architectures`.`name`) AS `pkgfile`," .
+    "`install_targets`.`name` AS `install_target`," .
+    "`repository_stabilities`.`name` AS `stability`," .
+    "IF(`binary_packages`.`is_to_be_deleted`,1,0) AS `is_to_be_deleted`" .
+    " FROM `binary_packages`" .
+    " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+    " AND `repositories`.`is_on_master_mirror`" .
+    " JOIN `repository_stabilities` ON `repositories`.`stability`=`repository_stabilities`.`id`" .
+    " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+    " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
+    " AND `dependency_types`.`relevant_for_binary_packages`" .
+    " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
+    " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+    " WHERE EXISTS (" .
+      "SELECT * FROM `install_target_providers`" .
+      " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
+    ")" .
+    " AND NOT EXISTS (" .
+      "SELECT * FROM `install_target_providers`" .
+      " JOIN `binary_packages` AS `prov_bp` ON `prov_bp`.`id`=`install_target_providers`.`package`" .
+      " JOIN `repositories` AS `prov_r` ON `prov_bp`.`repository`=`prov_r`.`id`" .
+      " JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`" .
+      " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
+      " AND `repositories`.`stability`=`repository_stability_relations`.`less_stable`" .
+    ") ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
+  ))
+    die($mysql -> error);
+
+  if ($result -> num_rows > 0) {
+    while ($row = $result->fetch_assoc()) {
+      if ($row["is_to_be_deleted"]==1)
+        print "<font color=\"#00ff00\">(marked as to-be-deleted) ";
+      else
+        print "<font color=\"#800000\">";
+      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package installable from enabled " . $row["stability"] . " repositories.<br>\n";
+      print "</font>";
+    }
+
+  }
+
+?>
+</body>
+</html>
-- 
cgit v1.2.3-54-g00ecf


From e40900974a9671651e0b18f4c9367e498157814c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 11:40:07 +0100
Subject: bin/get-package-updates: only put packages on the deletion-list which
 are known in the database

---
 bin/get-package-updates | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 0276b7e..b48660d 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -99,7 +99,16 @@ fi
 # delete_package package
 # mark $package for deletion
 delete_package() {
-  echo "$1" >> \
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
+    printf ' FROM `package_sources`'
+    mysql_join_package_sources_build_assignments
+    mysql_join_build_assignments_binary_packages
+    printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
+      "$(printf '%s' "$1" | base64 -w0)"
+  } | \
+    mysql_run_query >> \
     "${work_dir}/deletion-list.new"
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
   find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \
-- 
cgit v1.2.3-54-g00ecf


From ff8d40d2771eb8e98d03b2d0cf734bc7d2d5faab Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 12:25:40 +0100
Subject: bin/delete-packages: some start

---
 bin/delete-packages | 368 ++++++++++++++++------------------------------------
 1 file changed, 113 insertions(+), 255 deletions(-)

diff --git a/bin/delete-packages b/bin/delete-packages
index 26d323a..4675c79 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -69,18 +69,15 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
-tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX")
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
 # Create a lock file and a trap.
 
 if ! ${no_action}; then
 
-  exec 9> "${build_list_lock_file}"
-  if ! flock ${block_flag} 9; then
-    >&2 echo 'come back (shortly) later - I cannot lock build list.'
-    exit 0
-  fi
+#  exec 9> "${build_list_lock_file}"
+#  if ! flock ${block_flag} 9; then
+#    >&2 echo 'come back (shortly) later - I cannot lock build list.'
+#    exit 0
+#  fi
 
   exec 8> "${package_database_lock_file}"
   if ! flock ${block_flag} 8; then
@@ -96,260 +93,121 @@ if ! ${no_action}; then
 
 fi
 
-clean_up_lock_file() {
-  if ! ${no_action}; then
-    rm -f "${package_database_lock_file}" "${build_list_lock_file}"
-  fi
-  rm -rf --one-file-system "${tmp_dir}"
-}
-
-trap clean_up_lock_file EXIT
-
-cp \
-  "${work_dir}/deletion-list" \
-  "${work_dir}/build-list" \
-  "${tmp_dir}/"
-
-all_repos="${standalone_package_repositories} ${stable_package_repositories} ${testing_package_repositories} ${staging_package_repositories}"
-all_repos="core"
-
-for repo in ${all_repos}; do
-  mkdir "${tmp_dir}/${repo}"
-  ${master_mirror_rsync_command} \
-    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
-    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
-    "${tmp_dir}/${repo}/"
-done
-
-for repo in ${all_repos}; do
-  tar -C "${tmp_dir}/${repo}" -xzf "${tmp_dir}/${repo}/${repo}.db.tar.gz"
-done
+tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-find "${tmp_dir}" -mindepth 3 -maxdepth 3 -name 'desc' -exec \
+# shellcheck disable=SC2016
+{
+  printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+  printf 'INSERT IGNORE INTO `to_deletes`'
+  printf ' SELECT `binary_packages`.`id`'
+  printf ' FROM `binary_packages`'
+  mysql_join_binary_packages_dependencies
+  mysql_join_dependencies_dependency_types
+  printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+  mysql_join_binary_packages_repositories
+  printf ' AND `repositories`.`is_on_master_mirror`'
+  printf ' WHERE `binary_packages`.`is_to_be_deleted`'
+  printf ' AND NOT EXISTS ('
+    printf 'SELECT * FROM `install_target_providers`'
+    printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+  printf ');\n'
+
+  printf ' SELECT DISTINCT "repo",`repositories`.`name`'
+  printf ' FROM `to_deletes`'
+  printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  printf ';\n'
+
+  printf ' SELECT "package",`repositories`.`name`,`binary_packages`.`pkgname`'
+  printf ' FROM `to_deletes`'
+  printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  printf ';\n'
+
+  printf ' SELECT "package-id",`to_deletes`.`id`'
+  printf ' FROM `to_deletes`'
+  printf ';\n'
+} | \
+  mysql_run_query | \
   sed '
-    /^%DEPENDS%$/{
-      s/.*//
-      :dep_loop
-      N
-      /\n$/{
-        s@\(^\|\n\)\(.\)@\1depends \2@g
-        bend
-      }
-      bdep_loop
+    y/\t/ /
+    /^repo /{
+      s/^\S\+ //
+      w '"${tmp_dir}"'/repositories
+      d
     }
-    /^%PROVIDES%$/{
-      s/.*//
-      :pro_loop
-      N
-      /\n$/{
-        s@\(^\|\n\)\(.\)@\1provides \2@g
-        bend
-      }
-      bpro_loop
+    /^package /{
+      s/^\S\+ //
+      w '"${tmp_dir}"'/packages
+      d
     }
-    d
-    :end
-    s/[<>=]\S*\($\|\n\)/\1/g
-    s#\(^\|\n\)\(.\)#\1{} \2#g
-  ' {} \; \
-  -printf '%p provides %p\n' | \
-  sed '
-    /^$/d
-    s|^\S\+/\([^/ ]\+\)/\([^/ ]\+\)\(-[^-/ ]\+\)\{2\}/desc |\1 \2 |
-    s| \S\+/\([^/]\+\)\(-[^/-]\+\)\{2\}/desc$| \1|
-  ' | \
-  awk '{print $3 " " $1 " " $2 " " $4}' | \
-  sed -n '
-    /^provides /{
-      s/^provides //
-      w '"${tmp_dir}"'/db.provides
-      b
-    }
-    /^depends /{
-      s/^depends //
-      w '"${tmp_dir}"'/db.depends
-      b
+    /^package-id /{
+      s/^\S\+ //
+      w '"${tmp_dir}"'/package-ids
+      d
     }
   '
 
+if [ ! -s "${tmp_dir}/packages" ]; then
+  printf 'Nothing to delete.\n'
+  exit
+fi
 
+if ${no_action}; then
+  printf 'I would delete the following packages:\n'
+  sed '
+    y, ,/,
+    s/^/  /
+  ' "${tmp_dir}/packages"
+  printf 'From these repositories:\n'
+  sed '
+    s/^/  /
+  ' "${tmp_dir}/repositories"
+  exit
+fi
 
-echo 'OK' >&2
-exit 42
+mkdir "${tmp_dir}/repos"
 
-# # sanity check
-#
-# for ending in 'done' 'testing'; do
-#   if [ "${ending}" = 'testing' ] && \
-#     [ -z "${packages_to_stabilize}" ]; then
-#     # if nothing is to be untested, we don't care about duplicate
-#     # testing packages (and maybe an unstaging fixes this anyway)
-#     continue
-#   fi
-#   if [ -n "$(
-#     find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
-#       sed 's|\(\.[^.]\+\)\{4\}$||' | \
-#       sort | \
-#       uniq -d
-#     )" ]; then
-#     >&2 echo 'Removing duplicates not yet implemented:'
-#     find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
-#       sed 's|\(\.[^.]\+\)\{4\}$||' | \
-#       sort | \
-#       uniq -d
-#     exit 42
-#   fi
-# done
-#
-# # packages which are done
-#
-# find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
-#   sed '
-#     s|\.done$||
-#   ' | \
-#   sort -u > \
-#   "${tmp_dir}/done-packages"
-#
-# # packages still on the build-list
-#
-# tr ' ' '.' < \
-#   "${work_dir}/build-list" | \
-#   sort -u > \
-#   "${tmp_dir}/keep-packages"
-#
-# find "${work_dir}/package-infos" -name '*.groups' \
-#   -exec grep -qx 'base\(-devel\)\?' {} \; \
-#   -printf '%f\n' | \
-#   sed '
-#     s|\.groups$||
-#   ' | \
-#   sort -u > \
-#   "${tmp_dir}/base-packages"
-#
-# # no base / base-devel packages on the build list?
-# if [ -z "$(
-#     join -j 1 \
-#       "${tmp_dir}/base-packages" \
-#       "${tmp_dir}/keep-packages"
-#   )" ]; then
-#   # unstage all base / base-devel packages from staging
-#   cat "${tmp_dir}/base-packages" "${tmp_dir}/base-packages" "${tmp_dir}/keep-packages" | \
-#     sort | \
-#     uniq -u | \
-#     sponge "${tmp_dir}/keep-packages"
-# fi
-#
-# # find all dependencies of the unstageable packages
-# mv \
-#   "${tmp_dir}/keep-packages" \
-#   "${tmp_dir}/new-keep-packages"
-# touch "${tmp_dir}/keep-packages"
-#
-# while [ -s "${tmp_dir}/new-keep-packages" ]; do
-#
-#   cat "${tmp_dir}/new-keep-packages" "${tmp_dir}/keep-packages" | \
-#     sort -u | \
-#     sponge "${tmp_dir}/keep-packages"
-#
-#   sed '
-#     s|^|'"${work_dir}"'/package-infos/|
-#     s|$|.depends|
-#   ' "${tmp_dir}/keep-packages" | \
-#     xargs -r grep -HF '' | \
-#     sed '
-#       s|^.*/||
-#       s|\.depends:| |
-#     ' | \
-#     sort -u | \
-#     sort -k2,2 > \
-#     "${tmp_dir}/keep-packages.depends"
-#
-#   sed '
-#     s|^|'"${work_dir}"'/package-infos/|
-#     s|$|.builds|
-#   ' "${tmp_dir}/done-packages" | \
-#     xargs -r grep -HF '' | \
-#     sed '
-#       s|^.*/||
-#       s|\.builds:| |
-#     ' | \
-#     sort -u | \
-#     sort -k2,2 > \
-#     "${tmp_dir}/done-packages.builds"
-#
-#   join -j 2 -o 1.1 \
-#     "${tmp_dir}/done-packages.builds" \
-#     "${tmp_dir}/keep-packages.depends" | \
-#     sort -u > \
-#     "${tmp_dir}/new-keep-packages"
-#
-#   # "new" is only what has not been there before
-#   cat "${tmp_dir}/keep-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/new-keep-packages" | \
-#     sort | \
-#     uniq -u | \
-#     sponge "${tmp_dir}/new-keep-packages"
-#
-# done
-#
-# # unlock build list
-#
-# rm -f "${build_list_lock_file}"
-# flock -u 9
-#
-# clean_up_lock_file() {
-#   rm -rf --one-file-system "${tmp_dir}"
-#   rm -f "${package_database_lock_file}"
-# }
-#
-# # calculate unstageable packages from keep_packages and done_packages
-#
-# done_packages=$(
-#   cat \
-#     "${tmp_dir}/keep-packages" \
-#     "${tmp_dir}/keep-packages" \
-#     "${tmp_dir}/done-packages" | \
-#     sort | \
-#     uniq -u
-# )
-#
-# # move packages in packages_to_stabilize from *testing/ to the stable repos
-#
-# # shellcheck disable=SC2046
-# move_packages $(
-#
-#   for package in ${packages_to_stabilize}; do
-#
-#     if [ -z "${package}" ]; then
-#       continue
-#     fi
-#
-#     printf '%s/%s/%s\n' \
-#       "${package}" \
-#       "$(official_or_community "${package}" 'testing')" \
-#       "$(repository_of_package "${package}")"
-#
-#   done
-#
-# )
-#
-# # move packages from *staging to *testing
-#
-# # shellcheck disable=SC2046
-# move_packages $(
-#
-#   for package in ${done_packages}; do
-#
-#     if [ -z "${package}" ]; then
-#       continue
-#     fi
-#
-#     printf '%s/%s/%s\n' \
-#       "${package}" \
-#       "$(official_or_community "${package}" 'staging')" \
-#       "$(official_or_community "${package}" 'testing')"
-#
-#   done
-#
-# )
-#
-# clean_up_lock_file
+while read -r repo; do
+  ${master_mirror_rsync_command} \
+    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db.tar.gz" \
+    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files.tar.gz" \
+    "${tmp_dir}/repos/"
+  # shellcheck disable=SC2046
+  repo-remove "${tmp_dir}/repos/${repo}.db.tar.gz" \
+    $(
+      grep "^$(str_to_regex "${repo}") " "${tmp_dir}/packages" | \
+        cut -d' ' -f2
+    )
+  ${master_mirror_rsync_command} \
+    "${tmp_dir}/repos/${repo}.db.tar.gz" \
+    "${tmp_dir}/repos/${repo}.files.tar.gz" \
+    "${master_mirror_rsync_directory}/i686/${repo}/"
+done < \
+  "${tmp_dir}/repositories"
+
+# shellcheck disable=SC2016
+{
+  printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `to_deletes`;\n' \
+    "${tmp_dir}/package-ids"
+  printf 'DELETE `binary_packages` FROM `binary_packages` '
+  printf ' JOIN `to_deletes` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  printf ';\n'
+} | \
+  mysql_run_query
+
+tr ' ' '/' < \
+  "${tmp_dir}/packages" | \
+  sed '
+    p
+    s/$/.sig/
+  ' | \
+  sed '
+    s,^,rm "i686/,
+    s,$,",
+  ' | \
+  ${master_mirror_sftp_command}
-- 
cgit v1.2.3-54-g00ecf


From dbf60cd6d389c2c3c2d42b28ccb7403dcdeebe4f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 13:13:53 +0100
Subject: bin/delete-packages: clarify TODO

---
 bin/delete-packages | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/delete-packages b/bin/delete-packages
index 4675c79..7e843c5 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -13,7 +13,7 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: finish this
+# TODO: delete other to-be-deleted packages if asked to do so
 
 # shellcheck disable=SC2016
 usage() {
-- 
cgit v1.2.3-54-g00ecf


From 080cfc23228f76825805643079f1a14e2d82fa2b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 13:48:56 +0100
Subject: bin/delete-packages: ignore deletion-list and other abstract packages
 as install_target_providers

---
 bin/delete-packages | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/delete-packages b/bin/delete-packages
index 7e843c5..258e441 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -110,7 +110,11 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
   printf ' WHERE `binary_packages`.`is_to_be_deleted`'
   printf ' AND NOT EXISTS ('
     printf 'SELECT * FROM `install_target_providers`'
+    mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+    mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+    mysql_join_repositories_repository_stabilities 'prov_r'
     printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+    printf ' AND `repository_stabilities`.`name` NOT IN ("forbidden","virtual")'
   printf ');\n'
 
   printf ' SELECT DISTINCT "repo",`repositories`.`name`'
-- 
cgit v1.2.3-54-g00ecf


From 2a4617fd25cf56e2d81d90c0c2aff81bfd9e9823 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 13:50:24 +0100
Subject: bin/delete-packages: repair deletion of packages on the mirror

---
 bin/delete-packages | 40 ++++++++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/bin/delete-packages b/bin/delete-packages
index 258e441..8a4ea4a 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -129,6 +129,14 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
   mysql_join_binary_packages_repositories
   printf ';\n'
 
+  printf ' SELECT "package-file",`repositories`.`name`,'
+  mysql_package_name_query
+  printf ' FROM `to_deletes`'
+  printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  printf ';\n'
+
   printf ' SELECT "package-id",`to_deletes`.`id`'
   printf ' FROM `to_deletes`'
   printf ';\n'
@@ -146,6 +154,15 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
       w '"${tmp_dir}"'/packages
       d
     }
+    /^package-file /{
+      s/^\S\+ //
+      s, ,/,
+      s/$/.pkg.tar.xz/
+      w '"${tmp_dir}"'/package-files
+      s/$/.sig/
+      w '"${tmp_dir}"'/package-files
+      d
+    }
     /^package-id /{
       s/^\S\+ //
       w '"${tmp_dir}"'/package-ids
@@ -196,22 +213,21 @@ done < \
   printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
   printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `to_deletes`;\n' \
     "${tmp_dir}/package-ids"
-  printf 'DELETE `binary_packages` FROM `binary_packages` '
+
+  printf 'UPDATE `binary_packages` '
   printf ' JOIN `to_deletes` ON `to_deletes`.`id`=`binary_packages`.`id`'
   mysql_join_binary_packages_repositories
   mysql_join_binary_packages_architectures
-  printf ';\n'
+  printf ' SET `repository`=('
+    printf 'SELECT `repositories`.`id`'
+    printf ' FROM `repositories`'
+    printf ' WHERE `repositories`.`name`="deletion-list"'
+  printf ');\n'
 } | \
   mysql_run_query
 
-tr ' ' '/' < \
-  "${tmp_dir}/packages" | \
-  sed '
-    p
-    s/$/.sig/
-  ' | \
-  sed '
-    s,^,rm "i686/,
-    s,$,",
-  ' | \
+sed '
+  s,^,rm "i686/,
+  s,$,",
+' "${tmp_dir}/package-files" | \
   ${master_mirror_sftp_command}
-- 
cgit v1.2.3-54-g00ecf


From 3137ac0fe134369bc1f4805b00df09af01965022 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 15:15:58 +0100
Subject: bin/bootstrap-mysql: "build-support" should be "standalone" and also
 the least stable repository on the master mirror

---
 bin/bootstrap-mysql | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index b7e08ba..06bbd7f 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -689,17 +689,20 @@ fi
       'stable' 'stable' \
       'stable' 'testing' \
       'stable' 'staging' \
+      'stable' 'standalone' \
       'stable' 'unbuilt' \
       'stable' 'forbidden' \
       'testing' 'testing' \
       'testing' 'staging' \
+      'testing' 'standalone' \
       'testing' 'unbuilt' \
       'testing' 'forbidden' \
       'staging' 'staging' \
+      'staging' 'standalone' \
       'staging' 'unbuilt' \
       'staging' 'forbidden' \
       'unbuilt' 'forbidden' \
-      'build-support' 'build-support'
+      'standalone' 'standalone'
     printf ';\n'
   } | \
     sed 's| OR ;|;|'
-- 
cgit v1.2.3-54-g00ecf


From 578d17ab21ced3db7f145ea208c52cf97a6a4a91 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 21:05:42 +0100
Subject: bin/build-packages: set permissions to 777 before deleting old
 build-tmp-dir

---
 bin/build-packages | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index 0f0c841..ea346bc 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -264,7 +264,11 @@ while [ "${count}" -ne 0 ]; do
         -mindepth 1 \
         -maxdepth 1 \
         -name 'tmp.build-packages.??????' \
-        -exec rm -rf --one-file-system {} \;
+        -printf '%p\n' | \
+        while read -r old_tmp_dir; do
+          find "${old_tmp_dir}" -xdev -exec chmod 777 {} \;
+          rm -rf --one-file-system "${old_tmp_dir}"
+        done
       tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
       trap bail_out EXIT
 
-- 
cgit v1.2.3-54-g00ecf


From 470615922a8f76eca77b87902a02498a223c51bf Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 21:36:23 +0100
Subject: bin/build-packages: chmod 777 also during library search

---
 bin/build-packages | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/build-packages b/bin/build-packages
index ea346bc..1bc1206 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -452,6 +452,7 @@ while [ "${count}" -ne 0 ]; do
                 awk '{print $2}' | \
                 sort -u > \
                 "${pkgfile}.so.needs"
+              find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 {} \;
               rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
             done
           >&2 printf 'running namcap ...'
-- 
cgit v1.2.3-54-g00ecf


From 142f167138609950fb9cf1a282ad8f970a62c99a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Mar 2018 11:42:46 +0100
Subject: bin/build-packages: 4262105 tries to remove the wrong directories

---
 bin/build-packages | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index 1bc1206..1ac8dc3 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -260,7 +260,7 @@ while [ "${count}" -ne 0 ]; do
         recursively_umount_and_rm "${tmp_dir}"
         exit "${err}"
       }
-      find . \
+      find "${work_dir}" \
         -mindepth 1 \
         -maxdepth 1 \
         -name 'tmp.build-packages.??????' \
-- 
cgit v1.2.3-54-g00ecf


From 172370cb4be9d705f379f111cf895f67097a255b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Mar 2018 15:28:48 +0100
Subject: bin/db-update: --progressive new

---
 bin/db-update | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 73 insertions(+), 7 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 01a38ad..a5fa925 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -24,23 +24,32 @@ usage() {
   >&2 echo ' move possible packages from staging to testing.'
   >&2 echo ''
   >&2 echo 'possible options:'
-  >&2 echo '  -b|--block:       If necessary, wait for lock blocking.'
-  >&2 echo '  -h|--help:        Show this help and exit.'
-  >&2 echo '  -n|--no-action:   Only print what would be moved.'
+  >&2 echo '  -b|--block:'
+  >&2 echo '    If necessary, wait for lock blocking.'
+  >&2 echo '  -h|--help:'
+  >&2 echo '    Show this help and exit.'
+  >&2 echo '  -n|--no-action:'
+  >&2 echo '    Only print what would be moved.'
+  >&2 echo '  -p|--progressive:'
+  >&2 echo '    Move forward any package which replaces no package whose'
+  >&2 echo '    dependencies are all available somewhere.'
+  >&2 echo '    Note, that this _may_ move _less_ packages.'
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
 eval set -- "$(
-  getopt -o bhn \
+  getopt -o bhnp \
     --long block \
     --long help \
     --long no-action \
+    --long progressive \
     -n "$(basename "$0")" -- "$@" || \
   echo usage
 )"
 
 block_flag='-n'
 no_action=false
+progressive=false
 
 while true
 do
@@ -54,6 +63,9 @@ do
     -n|--no-action)
       no_action=true
     ;;
+    -p|--progressive)
+      progressive=true
+    ;;
     --)
       shift
       break
@@ -98,8 +110,62 @@ for source_stability in 'testing' 'staging'; do
 
   # shellcheck disable=SC2016
   {
-    printf 'CALL calculate_maximal_moveable_set("%s");\n' \
-      "${source_stability}"
+    if ${progressive}; then
+      printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \
+        'moveable' 'replaced'
+      printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+      printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+
+      printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+      printf ' SELECT `binary_packages`.`id`,`subst_bp`.`id`'
+      printf ' FROM `binary_packages`'
+      mysql_join_binary_packages_repositories
+      printf ' AND `repositories`.`is_on_master_mirror`'
+      mysql_join_binary_packages_build_assignments
+      mysql_join_build_assignments_package_sources
+      mysql_join_package_sources_upstream_repositories
+      mysql_join_upstream_repositories_repository_moves
+      printf ' AND `repository_moves`.`to_repository`=`binary_packages`.`repository`'
+      printf ' JOIN `binary_packages` AS `subst_bp`'
+      printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
+      printf ' AND `repository_moves`.`from_repository`=`subst_bp`.`repository`'
+      mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+      mysql_join_repositories_repository_stabilities 'subst_r' 'subst_rs'
+      printf ' AND `subst_rs`.`name`="%s"' \
+        "${source_stability}"
+      mysql_join_binary_packages_dependencies
+      mysql_join_dependencies_dependency_types
+      printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+      printf ' WHERE NOT EXISTS ('
+        printf 'SELECT * FROM `install_target_providers`'
+        printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+      printf ');\n'
+
+      printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+      printf ' SELECT `replaced_binary_packages`.`replaced_by`,`binary_packages`.`repository`'
+      printf ' FROM `replaced_binary_packages`'
+      printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`'
+      printf ';\n'
+
+      printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+      printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`'
+      printf ' FROM `binary_packages`'
+      mysql_join_binary_packages_repositories
+      printf ' AND `repositories`.`is_on_master_mirror`'
+      mysql_join_binary_packages_build_assignments
+      mysql_join_build_assignments_package_sources
+      mysql_join_package_sources_upstream_repositories
+      mysql_join_upstream_repositories_repository_moves
+      printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
+      printf ' WHERE NOT EXISTS ('
+        printf 'SELECT * FROM `binary_packages` AS `repl_bp`'
+        printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`'
+        printf ' AND `repl_bp`.`repository`=`repository_moves`.`to_repository`'
+      printf ');\n'
+    else
+      printf 'CALL calculate_maximal_moveable_set("%s");\n' \
+        "${source_stability}"
+    fi
 
     printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
     printf 'INSERT IGNORE INTO `rps` (`id`)'
@@ -143,7 +209,7 @@ for source_stability in 'testing' 'staging'; do
     mysql_join_binary_packages_architectures
     printf ';\n'
   } | \
-    mysql_run_query | \
+    mysql buildmaster -N --raw --batch | \
     tr '\t' ' ' | \
     grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
     while read -r what content; do
-- 
cgit v1.2.3-54-g00ecf


From 79a749a8d46ca2b8b018234da307490be79a3116 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Mar 2018 15:51:59 +0100
Subject: bin/db-update: do not move untested packages - this breaks stuff

---
 bin/db-update | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index a5fa925..2e8e229 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -133,6 +133,12 @@ for source_stability in 'testing' 'staging'; do
       mysql_join_repositories_repository_stabilities 'subst_r' 'subst_rs'
       printf ' AND `subst_rs`.`name`="%s"' \
         "${source_stability}"
+      # TODO: we may relax this condition, if we a) got rid of state files
+      # or b) improved keeping them in sync (*.testing -> /dev/null is not
+      # implemented, only .tested -> /dev/null)
+      if [ ! "${source_stability}" = 'staging' ]; then
+        printf ' AND `subst_bp`.`is_tested`'
+      fi
       mysql_join_binary_packages_dependencies
       mysql_join_dependencies_dependency_types
       printf ' AND `dependency_types`.`relevant_for_binary_packages`'
@@ -152,6 +158,15 @@ for source_stability in 'testing' 'staging'; do
       printf ' FROM `binary_packages`'
       mysql_join_binary_packages_repositories
       printf ' AND `repositories`.`is_on_master_mirror`'
+      mysql_join_repositories_repository_stabilities
+      printf ' AND `repository_stabilities`.`name`="%s"' \
+        "${source_stability}"
+      # TODO: we may relax this condition, if we a) got rid of state files
+      # or b) improved keeping them in sync (*.testing -> /dev/null is not
+      # implemented, only .tested -> /dev/null)
+      if [ ! "${source_stability}" = 'staging' ]; then
+        printf ' AND `binary_packages`.`is_tested`'
+      fi
       mysql_join_binary_packages_build_assignments
       mysql_join_build_assignments_package_sources
       mysql_join_package_sources_upstream_repositories
-- 
cgit v1.2.3-54-g00ecf


From fad367e19d2adb6662a5543645e77190e85e94e9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 13 Mar 2018 15:54:06 +0100
Subject: bin/db-update: trigger the mirror refreshs at the end, not in between

---
 bin/db-update | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 2e8e229..8a68346 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -341,8 +341,6 @@ for source_stability in 'testing' 'staging'; do
   done < \
     "${tmp_dir}/repositories"
 
-  trigger_mirror_refreshs
-
   # shellcheck disable=SC2016
   {
     printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
@@ -414,3 +412,5 @@ for source_stability in 'testing' 'staging'; do
     -not -exec test -s '{}' \; \
     -delete
 done
+
+trigger_mirror_refreshs
-- 
cgit v1.2.3-54-g00ecf


From 329bf724368aed075f28002cf2f1d40d8f325de8 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 06:38:15 +0100
Subject: bin/get-package-updates: remove old locked and blocked marker of
 _updated_ packages, too

---
 bin/get-package-updates | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index b48660d..081f457 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -317,6 +317,9 @@ done | \
     if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
       # new or modified PKGBUILD
       sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
+      find "${work_dir}/package-states" -regextype grep \
+        -regex ".*/$(str_to_regex "${package}")\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)" \
+        -delete
       # shellcheck disable=SC2154
       echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
         "${work_dir}/build-list.new"
-- 
cgit v1.2.3-54-g00ecf


From 47d7ec4ec95a771535fe15071b009a26b5b43b8d Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 06:39:21 +0100
Subject: bin/db-update: do not circumvent mysql query logging anymore

---
 bin/db-update | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/db-update b/bin/db-update
index 8a68346..1cbe3b6 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -224,7 +224,7 @@ for source_stability in 'testing' 'staging'; do
     mysql_join_binary_packages_architectures
     printf ';\n'
   } | \
-    mysql buildmaster -N --raw --batch | \
+    mysql_run_query | \
     tr '\t' ' ' | \
     grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
     while read -r what content; do
-- 
cgit v1.2.3-54-g00ecf


From cd4463bfdf66882031af0f2aed430d803ad1618a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 08:37:29 +0100
Subject: bin/get-package-updates: satisfy shellcheck

---
 bin/get-package-updates | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 081f457..1cf8782 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -318,7 +318,7 @@ done | \
       # new or modified PKGBUILD
       sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
       find "${work_dir}/package-states" -regextype grep \
-        -regex ".*/$(str_to_regex "${package}")\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)" \
+        -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \
         -delete
       # shellcheck disable=SC2154
       echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
-- 
cgit v1.2.3-54-g00ecf


From 2c75afd19063d02a725ae0cf0a8c3572b1f45a07 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 08:38:34 +0100
Subject: bin/ii-connect: remove unnecessary "-regextype" from "find"

---
 bin/ii-connect | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/ii-connect b/bin/ii-connect
index 867381e..973b227 100755
--- a/bin/ii-connect
+++ b/bin/ii-connect
@@ -39,7 +39,6 @@ fi
 if [ "$1" = 'watch' ]; then
   while pgrep -x 'ii' > /dev/null; do
     find "${irc_dir}" \
-      -regextype sed \
       -type f \
       -name 'out' \
       -printf '%p\n' | \
-- 
cgit v1.2.3-54-g00ecf


From 73409ac0d623289da164be28c78286434ab4706e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 10:43:01 +0100
Subject: bin/bootstrap-mysql: use mysql_run_query, too

---
 bin/bootstrap-mysql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 06bbd7f..d18247e 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -526,7 +526,7 @@ if [ ! "$1" = 'slim' ]; then
       'SHOW VIEW' 'buildmaster.*'
     printf 'FLUSH PRIVILEGES;\n'
   } | \
-    mysql -u root -p buildmaster
+    mysql_run_query -u root -p
 fi
 
 # shellcheck disable=SC2016
@@ -770,7 +770,7 @@ fi
   } | \
     sed 's|,;|;|'
 } | \
-  mysql buildmaster
+  mysql_run_query
 
 find "${work_dir}/package-states" \
   -maxdepth 1 \
-- 
cgit v1.2.3-54-g00ecf


From 254ba8ac7e63d024da55f459a9d9d76c65e8d3ce Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 10:52:58 +0100
Subject: bin/strict-bashism-check: add some check for "mysql buildmaster"

---
 bin/strict-bashism-check | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/bin/strict-bashism-check b/bin/strict-bashism-check
index fa073b8..1d7c2be 100755
--- a/bin/strict-bashism-check
+++ b/bin/strict-bashism-check
@@ -49,14 +49,27 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 git archive "${tree}" | \
   tar -C "${tmp_dir}" -x
 
+if ! cd "${tmp_dir}"; then
+  echo 'Cannot cd.'
+  exit 1
+fi
+
 errors=$(
-  cd "${tmp_dir}" || \
-    echo 'Cannot cd.'
   find bin conf lib -type f -not -executable -not -name '.gitignore'
 )
 
 if [ -n "${errors}" ]; then
-  >&2 echo 'Non-executable files found in bin/ or conf/:'
+  >&2 echo 'Non-executable files found in bin/, conf/ or lib/:'
+  >&2 echo "${errors}"
+  exit 1
+fi
+
+errors=$(
+  grep -r '\(\s\|^\)mysql buildmaster\($\|\s\)' bin lib conf
+)
+
+if [ -n "${errors}" ]; then
+  >&2 echo 'Style error: call "mysql_run_query" instead of "mysql buildmaster":'
   >&2 echo "${errors}"
   exit 1
 fi
@@ -74,8 +87,6 @@ fi
 
 if which shellcheck >/dev/null 2>&1; then
   errors=$(
-    cd "${tmp_dir}" || \
-      echo 'Cannot cd.'
     shellcheck -x bin/* conf/* 2>&1
   )
 
-- 
cgit v1.2.3-54-g00ecf


From 288aa9fb981e71a58f1df9d0a0257e6968422d06 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 13:36:52 +0100
Subject: bin/build-packages: ignore "opt" for libraries, but also check
 symlinks

---
 bin/build-packages | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index 1ac8dc3..edfa913 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -446,7 +446,8 @@ while [ "${count}" -ne 0 ]; do
             while read -r pkgfile; do
               mkdir "${package_content_dir}/${pkgfile##*/}"
               tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
-              find "${package_content_dir}/${pkgfile##*/}" -type f \
+              find "${package_content_dir}/${pkgfile##*/}" \
+                -name 'opt' -prune , \
                 -exec objdump -x '{}' \; 2>/dev/null | \
                 grep -w 'NEEDED' | \
                 awk '{print $2}' | \
-- 
cgit v1.2.3-54-g00ecf


From 8a1585389b3fa07fa94b4820d1cb99c1e2b0e98e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 14:48:21 +0100
Subject: bin/build-packages: *.c32 is not a shared library that is needed

---
 bin/build-packages | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/build-packages b/bin/build-packages
index edfa913..501917f 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -451,6 +451,7 @@ while [ "${count}" -ne 0 ]; do
                 -exec objdump -x '{}' \; 2>/dev/null | \
                 grep -w 'NEEDED' | \
                 awk '{print $2}' | \
+                grep -v '\.c32$' | \
                 sort -u > \
                 "${pkgfile}.so.needs"
               find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 {} \;
-- 
cgit v1.2.3-54-g00ecf


From 9d2577b5f5a23f41fb8af336e2a34cc96f63fb30 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 14:51:25 +0100
Subject: bin/seed-build-list: -a|--auto new to reschedule all packages with
 broken dependencies

---
 bin/seed-build-list | 48 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 2 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index d49a8f3..0ede139 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -14,6 +14,9 @@ usage() {
   >&2 echo '    - a list of packages which need to be rebuilt'
   >&2 echo ''
   >&2 echo 'possible options:'
+  >&2 echo '  -a|--auto:'
+  >&2 echo '    Automatically reschedule packages which have run-time'
+  >&2 echo '    dependencies that are not available anywhere.'
   >&2 echo '  -h|--help:'
   >&2 echo '    Show this help and exit.'
   >&2 echo '  -i|--ignore $package:'
@@ -32,7 +35,8 @@ tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
 trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT
 
 eval set -- "$(
-  getopt -o hi:m:np: \
+  getopt -o ahi:m:np: \
+    --long auto \
     --long help \
     --long ignore: \
     --long mirror: \
@@ -46,11 +50,15 @@ touch "${tmp_dir}/mirrors"
 touch "${tmp_dir}/package-regexes"
 touch "${tmp_dir}/ignore-packages"
 
+auto=false
 update=true
 
 while true
 do
   case "$1" in
+    -a|--auto)
+      auto=true
+    ;;
     -h|--help)
       usage 0
     ;;
@@ -89,7 +97,8 @@ if [ $# -ne 0 ]; then
 fi
 
 if [ ! -s "${tmp_dir}/mirrors" ] && \
-  [ ! -s "${tmp_dir}/package-regexes" ]; then
+  [ ! -s "${tmp_dir}/package-regexes" ] && \
+  ! ${auto}; then
   # nothing to do
   exit 0
 fi
@@ -249,6 +258,34 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
 # shellcheck disable=SC2016
 {
   cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages" "${tmp_dir}/pkgbases"
+  if ${auto}; then
+    printf 'CREATE TEMPORARY TABLE `bin_ids` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+    printf 'INSERT IGNORE INTO `bin_ids`'
+    printf ' SELECT `binary_packages`.`id` FROM `binary_packages`'
+    mysql_join_binary_packages_repositories
+    printf ' AND `repositories`.`is_on_master_mirror`'
+    printf ' AND `repositories`.`name`!="build-support"'
+    mysql_join_binary_packages_dependencies
+    mysql_join_dependencies_dependency_types
+    printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+    # nothing "less stable" has the same name
+    printf ' WHERE NOT EXISTS ('
+      printf 'SELECT * FROM `binary_packages` AS `subst_bp`'
+      mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+      printf ' AND `subst_r`.`name`!="build-support"'
+      printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`subst_r`.`stability`'
+      printf ' WHERE `repository_stability_relations`.`more_stable`=`repositories`.`stability`'
+      printf ' AND `subst_bp`.`id`!=`binary_packages`.`id`'
+      printf ' AND `subst_bp`.`pkgname`=`binary_packages`.`pkgname`'
+    printf ') AND NOT EXISTS ('
+      printf 'SELECT * FROM `install_target_providers`'
+      mysql_join_install_target_providers_binary_packages '' 'subst_bp'
+      mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+      printf ' AND `subst_r`.`name` NOT IN ("build-support","deletion-list","to-be-decided")'
+      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+    printf ')'
+    printf ';\n'
+  fi
   printf 'SELECT '
   printf '`pkgbases`.`pkgbase`,'
   printf '`git_repositories`.`head`,'
@@ -284,6 +321,13 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
       ' | \
       tr -d '\n'
   fi
+  if ${auto}; then
+    printf 'EXISTS ('
+      printf 'SELECT * FROM `bin_ids`'
+      printf ' WHERE `bin_ids`.`id`=`binary_packages`.`id`'
+    printf ')'
+    printf ' OR '
+  fi
   printf 'EXISTS ('
     printf 'SELECT * FROM `must_haves`'
     printf ' WHERE `must_haves`.`pkgname`=`binary_packages`.`pkgname`'
-- 
cgit v1.2.3-54-g00ecf


From 569fbf690cee01f6ca1bb039b11a3d6e73de878f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 15:09:58 +0100
Subject: bin/seed-build-list: insert into build-list _after_ inserting into
 database - in case something goes wrong

---
 bin/seed-build-list | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 0ede139..2edf041 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -348,11 +348,11 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
     flock -s 8
 
     while read -r pkgbase git_rev mod_git_rev repo; do
-      printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \
-        "${work_dir}/build-list"
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
       generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
+      printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \
+        "${work_dir}/build-list"
       printf '\n' >&2
     done
 
-- 
cgit v1.2.3-54-g00ecf


From 3d758f046fc29327d5ee57dcdadcb10f225d9dde Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 15:13:35 +0100
Subject: lib/mysql-functions: i -> _

---
 lib/mysql-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index aace33a..2a6eda8 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -34,7 +34,7 @@ mysql_run_query() {
     # we save the query in a file and delete that file if the query succeeded
     query_file=$(mktemp "${work_dir}/tmp.mysql-functions.query.$(date +'%Y-%m-%dT%T').XXXXXX")
     cat > "${query_file}"
-    for i in {1..10}; do
+    for _ in {1..10}; do
       {
         printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*"
         date
-- 
cgit v1.2.3-54-g00ecf


From e84d9650048d1dccf8d30ae456811a08ea88baaa Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 15:35:54 +0100
Subject: lib/mysql-functions: strip sub_pkgrel from extracted pkgrel

---
 lib/mysql-functions | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index 2a6eda8..4e0ce93 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -395,6 +395,7 @@ mysql_generate_package_metadata() {
         pkgrel=$(
           grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
             cut -d' ' -f3 | \
+            sed 's/\.[0-9]\+$//' | \
             base64_encode_each
         )
         epoch=$(
-- 
cgit v1.2.3-54-g00ecf


From 09fee973260d9d2c944e6538d46be157aae74236 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 15:47:35 +0100
Subject: bin/strict-bashism-check: check lib/, too - except opcode_list

---
 bin/strict-bashism-check | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bin/strict-bashism-check b/bin/strict-bashism-check
index 1d7c2be..cbf7e5b 100755
--- a/bin/strict-bashism-check
+++ b/bin/strict-bashism-check
@@ -87,7 +87,11 @@ fi
 
 if which shellcheck >/dev/null 2>&1; then
   errors=$(
-    shellcheck -x bin/* conf/* 2>&1
+    find bin conf lib \
+      -type f \
+      -not -name 'opcode_list' \
+      -not -name '.*' \
+      -exec shellcheck -x '{}' \; 2>&1
   )
 
   if [ -n "${errors}" ]; then
-- 
cgit v1.2.3-54-g00ecf


From 88dbd44bcc25d3a59e721915b887daa39acc0b66 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 15:51:46 +0100
Subject: lib/mysql-functions mysql_generate_package_metadata: create
 dependency of $x-doc packages on $x-$pkgver and install_target_provider $x ->
 $x-$pkgver, $x-$pkgver-$pkgrel

---
 lib/mysql-functions | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/lib/mysql-functions b/lib/mysql-functions
index 4e0ce93..b9ae4ab 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -535,6 +535,15 @@ mysql_generate_package_metadata() {
               s|^| (from_base64("|
               s|$|")),|
             '
+          printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"),"-",from_base64("%s"))),\n' \
+            "${pkgname}" \
+            "${epoch}" \
+            "${pkgver}" \
+            "${pkgrel}"
+          printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"))),\n' \
+            "${pkgname}" \
+            "${epoch}" \
+            "${pkgver}"
           printf ' (from_base64("%s"));\n' \
             "${pkgname}"
 
@@ -546,6 +555,14 @@ mysql_generate_package_metadata() {
               s|^| (from_base64("|
               s|$|")),|
             '
+          if printf '%s' "${pkgname}" | \
+            base64 -d | \
+            grep -q -- '-doc$'; then
+            printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"))),\n' \
+              "$(printf '%s' "${pkgname}" | base64 -d | sed 's/-doc$//' | base64 -w0)" \
+              "${epoch}" \
+              "${pkgver}"
+          fi
           printf ' ("base");\n'
 
           echo "${checkdepends}" | \
-- 
cgit v1.2.3-54-g00ecf


From 8abcf920222da35737f5739b32b0bfd76e9bff4f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 16:43:04 +0100
Subject: bin/delete-packages: several bug fixes and improvements

---
 bin/delete-packages | 53 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/bin/delete-packages b/bin/delete-packages
index 8a4ea4a..4343bc7 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -157,7 +157,6 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
     /^package-file /{
       s/^\S\+ //
       s, ,/,
-      s/$/.pkg.tar.xz/
       w '"${tmp_dir}"'/package-files
       s/$/.sig/
       w '"${tmp_dir}"'/package-files
@@ -175,19 +174,6 @@ if [ ! -s "${tmp_dir}/packages" ]; then
   exit
 fi
 
-if ${no_action}; then
-  printf 'I would delete the following packages:\n'
-  sed '
-    y, ,/,
-    s/^/  /
-  ' "${tmp_dir}/packages"
-  printf 'From these repositories:\n'
-  sed '
-    s/^/  /
-  ' "${tmp_dir}/repositories"
-  exit
-fi
-
 mkdir "${tmp_dir}/repos"
 
 while read -r repo; do
@@ -195,19 +181,50 @@ while read -r repo; do
     "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db.tar.gz" \
     "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files.tar.gz" \
     "${tmp_dir}/repos/"
+
+  tar -Oxzf "${tmp_dir}/repos/${repo}.db.tar.gz" --wildcards '*/desc' | \
+    sed -n '
+      /^%FILENAME%$/{
+        N
+        s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/
+        T
+        p
+      }
+    ' | \
+    while read -r old new; do
+      printf 's,/%s\\(\\.sig\\)\\?$,/%s\\1,\n' \
+        "$(str_to_regex "${old}")" \
+        "$(str_to_regex "${new}")"
+    done >> \
+    "${tmp_dir}/sub_pkgrel-removal.sed"
+
   # shellcheck disable=SC2046
   repo-remove "${tmp_dir}/repos/${repo}.db.tar.gz" \
     $(
       grep "^$(str_to_regex "${repo}") " "${tmp_dir}/packages" | \
         cut -d' ' -f2
     )
-  ${master_mirror_rsync_command} \
-    "${tmp_dir}/repos/${repo}.db.tar.gz" \
-    "${tmp_dir}/repos/${repo}.files.tar.gz" \
-    "${master_mirror_rsync_directory}/i686/${repo}/"
+  if ! ${no_action}; then
+    ${master_mirror_rsync_command} \
+      "${tmp_dir}/repos/${repo}.db.tar.gz" \
+      "${tmp_dir}/repos/${repo}.files.tar.gz" \
+      "${master_mirror_rsync_directory}/i686/${repo}/"
+  fi
 done < \
   "${tmp_dir}/repositories"
 
+if [ -s "${tmp_dir}/sub_pkgrel-removal.sed" ]; then
+  sed -i -f "${tmp_dir}/sub_pkgrel-removal.sed" "${tmp_dir}/package-files"
+fi
+
+if ${no_action}; then
+  printf 'Now, I would remove the packages from the database and delete the following files from the master mirror:\n'
+  sed '
+    s,^,  ,
+  ' "${tmp_dir}/package-files"
+  exit
+fi
+
 # shellcheck disable=SC2016
 {
   printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
-- 
cgit v1.2.3-54-g00ecf


From d39a7bdecd340164c04965be17b51f7ad6de0c5c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 16:48:36 +0100
Subject: bin/sanity-check: wait 1sec before checking for left over
 tmp.mysql-functions.query files

---
 bin/sanity-check | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/sanity-check b/bin/sanity-check
index 824b7a2..b3e02aa 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -400,6 +400,9 @@ while [ $# -gt 0 ]; do
         i_am_insane
       fi
 
+      # hopefully, this gets rid of false positives :-)
+      sleep 1
+
       if find "${work_dir}" -mindepth 1 -maxdepth 1 -name 'tmp.mysql-functions.query.*' | \
         grep '\S' >> \
         "${tmp_dir}/messages"; then
-- 
cgit v1.2.3-54-g00ecf


From 4264522c07923f306e119eed1b720eb82901ec36 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 10:29:01 +0100
Subject: web-scripts/mysql-issues.php: add option to ignore haskell packages

---
 web-scripts/mysql-issues.php | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/web-scripts/mysql-issues.php b/web-scripts/mysql-issues.php
index 56d4c27..fc948a7 100755
--- a/web-scripts/mysql-issues.php
+++ b/web-scripts/mysql-issues.php
@@ -7,6 +7,11 @@
 <a href="/">Start page</a><br>
 <?php
 
+  if (isset($_GET["ignore-haskell"]))
+    $ignore = " AND `install_targets`.`name` NOT LIKE \"libHS%\"";
+  else
+    $ignore = "";
+
   $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
   if ($mysql->connect_error) {
     die("Connection failed: " . $mysql->connect_error);
@@ -34,7 +39,9 @@
     " WHERE NOT EXISTS (" .
       "SELECT * FROM `install_target_providers`" .
       " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
-    ") ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
+    ")" .
+    $ignore .
+    " ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
   ))
     die($mysql -> error);
 
@@ -83,7 +90,9 @@
       " JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`" .
       " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
       " AND `repositories`.`stability`=`repository_stability_relations`.`less_stable`" .
-    ") ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
+    ")" .
+    $ignore .
+    " ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
   ))
     die($mysql -> error);
 
-- 
cgit v1.2.3-54-g00ecf


From c31b7cc1dd5bd9ccce5b9cd22b68c5514ba40455 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 10:39:42 +0100
Subject: bin/build-packages: do not check "any" packages for linked libraries

---
 bin/build-packages | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/bin/build-packages b/bin/build-packages
index 501917f..1a73409 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -444,18 +444,23 @@ while [ "${count}" -ne 0 ]; do
           find "${tar_content_dir}" -maxdepth 1 \
             -name '*.pkg.tar.xz' | \
             while read -r pkgfile; do
-              mkdir "${package_content_dir}/${pkgfile##*/}"
-              tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
-              find "${package_content_dir}/${pkgfile##*/}" \
-                -name 'opt' -prune , \
-                -exec objdump -x '{}' \; 2>/dev/null | \
-                grep -w 'NEEDED' | \
-                awk '{print $2}' | \
-                grep -v '\.c32$' | \
+              if printf '%s\n' "${pkgfile}" | \
+                grep -vq '-any\.pkg\.tar\.xz$'; then
+                # we do not check "any" packages for linked libraries
+                # (why do they have them in the first place?)
+                mkdir "${package_content_dir}/${pkgfile##*/}"
+                tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
+                find "${package_content_dir}/${pkgfile##*/}" \
+                  -name 'opt' -prune , \
+                  -exec objdump -x '{}' \; 2>/dev/null | \
+                  grep -w 'NEEDED' | \
+                  awk '{print $2}' | \
+                  grep -v '\.c32$'
+                find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 '{}' \;
+                rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
+              fi | \
                 sort -u > \
                 "${pkgfile}.so.needs"
-              find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 {} \;
-              rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
             done
           >&2 printf 'running namcap ...'
           if [ "${repository}" = 'multilib' ]; then
-- 
cgit v1.2.3-54-g00ecf


From a0ebdbe495c23d2cbbea2bc0e0f85815be5e364f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 10:50:36 +0100
Subject: bin/build-packages: remove paths from required libraries

---
 bin/build-packages | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index 1a73409..a1a0475 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -455,7 +455,10 @@ while [ "${count}" -ne 0 ]; do
                   -exec objdump -x '{}' \; 2>/dev/null | \
                   grep -w 'NEEDED' | \
                   awk '{print $2}' | \
-                  grep -v '\.c32$'
+                  sed '
+                    /\.c32$/d
+                    s,^.*/,,
+                  '
                 find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 '{}' \;
                 rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
               fi | \
-- 
cgit v1.2.3-54-g00ecf


From 912971f958e21dcc80b53af611c0e0cec8454e57 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 11:17:38 +0100
Subject: bin/modify-package-state: lock package database

---
 bin/modify-package-state | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 9b0571b..6a6716f 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -128,6 +128,12 @@ if ! flock -s -n 9; then
   exit
 fi
 
+exec 8> "${package_database_lock_file}"
+if ! flock -n 8; then
+  >&2 echo 'Cannot get package-database lock.'
+  exit
+fi
+
 while read -r package reason; do
   case "${action}" in
     'faulty'|'tested')
-- 
cgit v1.2.3-54-g00ecf


From 3595bae3abb8514bb150e11094fd1ece763c4c46 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 11:49:27 +0100
Subject: bin/bootstrap-mysql: bugfix: stability of source repository was used
 instead of target repository

---
 bin/bootstrap-mysql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index d18247e..22603fc 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -393,7 +393,7 @@ if [ ! "$1" = 'slim' ]; then
     printf ' RIGHT JOIN `moveable_binary_packages`'
     printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`'
     printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
-    mysql_join_binary_packages_repositories
+    printf ' JOIN `target_repositories` ON `moveable_binary_packages`.`to_repository`=`repositories`.`id`'
     mysql_join_binary_packages_dependencies
     mysql_join_dependencies_dependency_types
     printf ' AND `dependency_types`.`relevant_for_binary_packages`'
@@ -404,7 +404,7 @@ if [ ! "$1" = 'slim' ]; then
       mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
       printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`less_stable`'
       printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
-      printf ' AND `repositories`.`stability`=`repository_stability_relations`.`more_stable`'
+      printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`more_stable`'
       printf ' AND NOT EXISTS ('
         printf 'SELECT * FROM `replaced_binary_packages_copy`'
         printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`'
-- 
cgit v1.2.3-54-g00ecf


From fcf305cd0df5cb0ef5e13d0867b1e57ea8eff1fb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 12:03:00 +0100
Subject: bin/bootstrap-mysql: fix typo

---
 bin/bootstrap-mysql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 22603fc..87ac313 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -393,7 +393,7 @@ if [ ! "$1" = 'slim' ]; then
     printf ' RIGHT JOIN `moveable_binary_packages`'
     printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`'
     printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
-    printf ' JOIN `target_repositories` ON `moveable_binary_packages`.`to_repository`=`repositories`.`id`'
+    printf ' JOIN `repositories` AS `target_repositories` ON `moveable_binary_packages`.`to_repository`=`target_repositories`.`id`'
     mysql_join_binary_packages_dependencies
     mysql_join_dependencies_dependency_types
     printf ' AND `dependency_types`.`relevant_for_binary_packages`'
-- 
cgit v1.2.3-54-g00ecf


From 96193bd8db538d68d2324c1d97e1118c63d0582f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 14:07:05 +0100
Subject: bin/bootstrap-mysql: bugfix and new TODO

---
 bin/bootstrap-mysql | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 87ac313..bd6df9c 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -308,13 +308,18 @@ if [ ! "$1" = 'slim' ]; then
     # Give a maximal list of packages to be moved, while implementing the
     # condition from db-update:
 
-    # Every package which is replaced, must have its provided install_targets:
+    # Every package which is replaced[1], must have its provided install_targets:
     #   a) provided by another moved or not-replaced package or
     #   b) not required by any not-replaced package.
 
     # Every package being moved needs to have all dependencies
     # installable in the target repository.
 
+    # TODO: [1] A "replaced" package may also be in a different repository
+    # e.g. if a-2 is moved from [staging] to [testing] and there is only
+    # a-1 in [core], then this will be "replaced" by a-2 on a system
+    # running on [testing] repositories.
+
     printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n'
     printf 'DELIMITER //\n'
     printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n'
@@ -402,9 +407,9 @@ if [ ! "$1" = 'slim' ]; then
       printf 'SELECT * FROM `install_target_providers`'
       mysql_join_install_target_providers_binary_packages '' 'prov_bp'
       mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
-      printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`less_stable`'
+      printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`'
       printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
-      printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`more_stable`'
+      printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`less_stable`'
       printf ' AND NOT EXISTS ('
         printf 'SELECT * FROM `replaced_binary_packages_copy`'
         printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`'
-- 
cgit v1.2.3-54-g00ecf


From cdd7dfd2b945f46128563b02123a3cb2f6cdf78e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 14:59:27 +0100
Subject: bin/db-update --force new

---
 bin/db-update | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 55 insertions(+), 5 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 1cbe3b6..129a89d 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -26,6 +26,8 @@ usage() {
   >&2 echo 'possible options:'
   >&2 echo '  -b|--block:'
   >&2 echo '    If necessary, wait for lock blocking.'
+  >&2 echo '  -f|--force $package-id:'
+  >&2 echo '    Force movement of Package with given id and move nothing else.'
   >&2 echo '  -h|--help:'
   >&2 echo '    Show this help and exit.'
   >&2 echo '  -n|--no-action:'
@@ -38,8 +40,9 @@ usage() {
 }
 
 eval set -- "$(
-  getopt -o bhnp \
+  getopt -o bf:hnp \
     --long block \
+    --long force \
     --long help \
     --long no-action \
     --long progressive \
@@ -50,6 +53,7 @@ eval set -- "$(
 block_flag='-n'
 no_action=false
 progressive=false
+force_ids=''
 
 while true
 do
@@ -57,6 +61,14 @@ do
     -b|--block)
       block_flag=''
     ;;
+    -f|--force)
+      shift
+      force_ids=$(
+        printf '%s' "$1" | \
+          base64 -w0
+        printf '\n%s' "${force_ids}"
+      )
+    ;;
     -h|--help)
       usage 0
     ;;
@@ -83,6 +95,12 @@ if [ $# -ne 0 ]; then
   usage
 fi
 
+if ${progressive} && \
+  [ -n "${force_ids}" ]; then
+  >&2 echo 'db-update: conflicting arguments'
+  usage
+fi
+
 if [ -s "${work_dir}/build-master-sanity" ]; then
   >&2 echo 'Build master is not sane.'
   exit
@@ -90,9 +108,6 @@ fi
 
 # Create tmp_dir, lock and trap.
 
-tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
 exec 9> "${package_database_lock_file}"
 if ! flock ${block_flag} 9; then
   >&2 echo 'come back (shortly) later - I cannot lock package database.'
@@ -105,12 +120,47 @@ if ! flock -s ${block_flag} 8; then
   exit 0
 fi
 
+tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
 for source_stability in 'testing' 'staging'; do
   find "${tmp_dir}" -mindepth 1 -delete
 
   # shellcheck disable=SC2016
   {
-    if ${progressive}; then
+    if [ -n "${force_ids}" ]; then
+      printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \
+        'moveable' 'replaced'
+      printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+      printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+      printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+      printf ' VALUES'
+      # shellcheck disable=SC2086
+      printf '(from_base64("%s"),NULL),' \
+        ${force_ids} | \
+        sed 's/,$/;\n/'
+      printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`'
+      printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+      mysql_join_binary_packages_repositories
+      mysql_join_repositories_repository_stabilities
+      printf ' WHERE `repository_stabilities`.`name`!="%s";\n' \
+        "${source_stability}"
+      printf 'UPDATE `moveable_binary_packages`'
+      printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+      mysql_join_binary_packages_build_assignments
+      mysql_join_build_assignments_package_sources
+      mysql_join_package_sources_upstream_repositories
+      mysql_join_upstream_repositories_repository_moves
+      printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
+      printf ' SET `moveable_binary_packages`.`to_repository`=`repository_moves`.`to_repository`;\n'
+      printf 'DELETE FROM `moveable_binary_packages` WHERE `moveable_binary_packages`.`to_repository` IS NULL;\n'
+      printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+      printf ' SELECT `binary_packages`.`id`,`moveable_binary_packages`.`id`'
+      printf ' FROM `moveable_binary_packages`'
+      printf ' JOIN `binary_packages` AS `subst_bp` ON `moveable_binary_packages`.`id`=`subst_bp`.`id`'
+      printf ' JOIN `binary_packages` ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
+      printf ' AND `binary_packages`.`repository`=`moveable_binary_packages`.`to_repository`;\n'
+    elif ${progressive}; then
       printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \
         'moveable' 'replaced'
       printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
-- 
cgit v1.2.3-54-g00ecf


From 6641f49081fe9b8d59953c2a96fff25ae4403f48 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 15:09:47 +0100
Subject: bin/build-packages: bugfix

---
 bin/build-packages | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-packages b/bin/build-packages
index a1a0475..c7b5f85 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -445,7 +445,7 @@ while [ "${count}" -ne 0 ]; do
             -name '*.pkg.tar.xz' | \
             while read -r pkgfile; do
               if printf '%s\n' "${pkgfile}" | \
-                grep -vq '-any\.pkg\.tar\.xz$'; then
+                grep -vq -- '-any\.pkg\.tar\.xz$'; then
                 # we do not check "any" packages for linked libraries
                 # (why do they have them in the first place?)
                 mkdir "${package_content_dir}/${pkgfile##*/}"
-- 
cgit v1.2.3-54-g00ecf


From 2747cf135820de7334144fd4e4a81c458093a08f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 15:47:09 +0100
Subject: bin/build-master-status: rely less on state files

---
 bin/build-master-status | 174 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 116 insertions(+), 58 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 006750b..aa15fd4 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -64,63 +64,93 @@ tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
 stable=$(
-  ls_master_mirror 'i686' | \
-    grep -v 'testing$\|staging$\|-unstable$' | \
-    while read -r dir; do
-      ls_master_mirror "i686/${dir}"
-    done | \
-    grep -c '\.pkg\.tar\.xz$'
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="stable";\n'
+  } | \
+    mysql_run_query
 )
 tasks=$(
-  grep -c '^\S\+ \S\+ \S\+ \S\+$' \
-    "${work_dir}/build-list"
-) || true
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list";\n'
+  } | \
+    mysql_run_query
+)
 pending_packages=$(
-  grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \
-    tr ' ' '.' | \
-    while read -r package; do
-      generate_package_metadata "${package}" 2>&1 > /dev/null
-      cat "${work_dir}/package-infos/${package}.packages"
-    done |
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list";\n'
+  } | \
+    mysql_run_query
 )
 next_tasks=$(
+  # shellcheck disable=SC2016
   {
-    cat "${work_dir}/build-list"
-    find "${work_dir}/package-states" -maxdepth 1 \
-      \( -name '*.broken' -o -name '*.blocked' \) \
-      -printf '%f\n' | \
-      sed '
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
-        p
-      '
+    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND NOT EXISTS ('
+      printf 'SELECT * FROM `dependencies`'
+      mysql_join_dependencies_dependency_types
+      printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+      mysql_join_dependencies_install_target_providers
+      mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+      mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+      printf ' WHERE `prov_r`.`name`="build-list"'
+      printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`'
+    printf ')'
+    printf ';\n'
   } | \
-    sort | \
-    uniq -u | \
-    while read -r package git_revision mod_git_revision repository; do
-      if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
-        echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-      fi
-    done | \
-    wc -l
+    mysql_run_query
 )
 staging=$(
-  find "${work_dir}/package-states" -name '*.done' \
-    -exec cat '{}' \; | \
-    sort -u | \
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="staging";\n'
+  } | \
+    mysql_run_query
 )
 testing=$(
-  find "${work_dir}/package-states" -name '*.testing' \
-    -exec cat '{}' \; | \
-    sort -u | \
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="testing"'
+    printf ' AND NOT `binary_packages`.`is_tested`;\n'
+  } | \
+    mysql_run_query
 )
 tested=$(
-  find "${work_dir}/package-states" -name '*.tested' \
-    -exec cat '{}' \; | \
-    sort -u | \
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="testing"'
+    printf ' AND `binary_packages`.`is_tested`;\n'
+  } | \
+    mysql_run_query
 )
 {
   find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
@@ -145,28 +175,56 @@ tested=$(
   sort -u > \
   "${tmp_dir}/broken-packages-names"
 broken=$(
-  wc -l < \
-    "${tmp_dir}/broken-packages-names"
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_assignments`.`is_broken`;\n'
+  } | \
+    mysql_run_query
 )
 blocked=$(
-  find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_assignments`.`is_blocked` IS NOT NULL;\n'
+  } | \
+    mysql_run_query
 )
 locked=$(
-  find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM'
+    printf ' `build_slaves`'
+    mysql_join_build_slaves_build_assignments
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list";\n'
+  } | \
+    mysql_run_query
 )
 loops=$(
-  find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-    -regex '.*/loop_[0-9]\+' | \
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM'
+    printf ' `build_dependency_loops`'
+  } | \
+    mysql_run_query
 )
 looped_packages=$(
-  find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-    -regex '.*/loop_[0-9]\+' \
-    -exec cat '{}' \; | \
-    sort -u | \
-    wc -l
+  # shellcheck disable=SC2016
+  {
+    printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM'
+    printf ' `build_dependency_loops`'
+  } | \
+    mysql_run_query
 )
 
 {
-- 
cgit v1.2.3-54-g00ecf


From faca6282d6431986165cf90386175c2077a87481 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 15:55:21 +0100
Subject: bin/modify-package-state: add line breaks

---
 bin/modify-package-state | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 6a6716f..36fee85 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -101,7 +101,7 @@ input_file="$1"
 if ${report}; then
   if ! [ -w "${input_file}" ]; then
     >&2 printf \
-      'Cannot open file "%s" for writing.' \
+      'Cannot open file "%s" for writing.\n' \
       "${input_file}"
     exit 2
   fi
@@ -117,7 +117,7 @@ fi
 
 if ! [ -r "${input_file}" ]; then
   >&2 printf \
-    'Cannot open file "%s" for reading.' \
+    'Cannot open file "%s" for reading.\n' \
     "${input_file}"
   exit 2
 fi
-- 
cgit v1.2.3-54-g00ecf


From 0b6bf4babcdc819764aa961294af89240695eee2 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 15 Mar 2018 16:40:21 +0100
Subject: bin/return-assignment: accept 128 char libraries

---
 bin/return-assignment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index 717a30e..a61460e 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -455,7 +455,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_
 # shellcheck disable=SC2016
 {
   for lib_link in 'pl:provides' 'nl:needs'; do
-    printf 'CREATE TEMPORARY TABLE `%s` (`pkgfile` VARCHAR(64), `lib` VARCHAR(64));\n' \
+    printf 'CREATE TEMPORARY TABLE `%s` (`pkgfile` VARCHAR(64), `lib` VARCHAR(128));\n' \
       "${lib_link%:*}"
     find . -maxdepth 1 -name '*.pkg.tar.xz.so.'"${lib_link#*:}"'.gz' -execdir zgrep -HF '' '{}' \; | \
       sed -n '
-- 
cgit v1.2.3-54-g00ecf


From c058238e43130a9323de3bfde29639b747cd6578 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 16 Mar 2018 14:45:52 +0100
Subject: web-scripts/mysql-issues.php: add summary lines

---
 web-scripts/mysql-issues.php | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/web-scripts/mysql-issues.php b/web-scripts/mysql-issues.php
index fc948a7..8de8937 100755
--- a/web-scripts/mysql-issues.php
+++ b/web-scripts/mysql-issues.php
@@ -45,6 +45,8 @@
   ))
     die($mysql -> error);
 
+  print "Found " . ($result -> num_rows) . " serious issues.<br>\n";
+
   if ($result -> num_rows > 0) {
 
     while ($row = $result->fetch_assoc()) {
@@ -52,8 +54,8 @@
         print "<font color=\"#00ff00\">(marked as to-be-deleted) ";
       else
         print "<font color=\"#ff0000\">";
-      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package.<br>\n";
-      print "</font>";
+      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package.<br>";
+      print "</font>\n";
     }
 
   }
@@ -96,14 +98,16 @@
   ))
     die($mysql -> error);
 
+  print "Found " . ($result -> num_rows) . " stability issues.<br>\n";
+
   if ($result -> num_rows > 0) {
     while ($row = $result->fetch_assoc()) {
       if ($row["is_to_be_deleted"]==1)
         print "<font color=\"#00ff00\">(marked as to-be-deleted) ";
       else
         print "<font color=\"#800000\">";
-      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package installable from enabled " . $row["stability"] . " repositories.<br>\n";
-      print "</font>";
+      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package installable from enabled " . $row["stability"] . " repositories.<br>";
+      print "</font>\n";
     }
 
   }
-- 
cgit v1.2.3-54-g00ecf


From 861171fbba068d6a88ef44b472771ea582e17a90 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 17 Mar 2018 11:21:51 +0100
Subject: bin/seed-build-list: ignore packages also when chosen by pkgbase

---
 bin/seed-build-list | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 2edf041..341ccc6 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -297,6 +297,13 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
   printf ' FROM `pkgbases`'
   printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`'
   mysql_join_upstream_repositories_git_repositories
+  printf ' WHERE NOT EXISTS ('
+    printf 'SELECT * FROM `ignore_packages`'
+    printf ' JOIN `binary_packages` AS `i_bp` ON `ignore_packages`.`pkgname`=`i_bp`.`pkgname`'
+    mysql_join_binary_packages_build_assignments 'i_bp' 'i_ba'
+    mysql_join_build_assignments_package_sources 'i_ba' 'i_ps'
+    printf ' WHERE `i_ps`.`pkgbase`=`pkgbases`.`pkgbase`'
+  printf ');\n'
   printf ';\n'
   printf 'SELECT '
   printf '`package_sources`.`pkgbase`,'
-- 
cgit v1.2.3-54-g00ecf


From 66914531d268cb27a7a70aa747eb11625ee1a098 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 17 Mar 2018 18:05:13 +0100
Subject: web-scripts/mysql-issues.php: show by which package a broken package
 can be replaced

---
 web-scripts/mysql-issues.php | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/web-scripts/mysql-issues.php b/web-scripts/mysql-issues.php
index 8de8937..485bc21 100755
--- a/web-scripts/mysql-issues.php
+++ b/web-scripts/mysql-issues.php
@@ -27,7 +27,8 @@
     "`binary_packages`.`sub_pkgrel`,\"-\"," .
     "`architectures`.`name`) AS `pkgfile`," .
     "`install_targets`.`name` AS `install_target`," .
-    "IF(`binary_packages`.`is_to_be_deleted`,1,0) AS `is_to_be_deleted`" .
+    "IF(`binary_packages`.`is_to_be_deleted`,1,0) AS `is_to_be_deleted`," .
+    "`subst_r`.`name` AS `subst_repository`" .
     " FROM `binary_packages`" .
     " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
     " AND `repositories`.`is_on_master_mirror`" .
@@ -36,6 +37,13 @@
     " AND `dependency_types`.`relevant_for_binary_packages`" .
     " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
     " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+    " LEFT JOIN (`binary_packages` AS `subst_bp`" .
+    " JOIN `repositories` AS `subst_r` ON `subst_bp`.`repository`=`subst_r`.`id`" .
+    " JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`subst_r`.`id`" .
+    ")" .
+    " ON `subst_bp`.`pkgname`=`binary_packages`.`pkgname`" .
+    " AND `subst_bp`.`id`!=`binary_packages`.`id`" .
+    " AND `repository_stability_relations`.`more_stable`=`repositories`.`id`" .
     " WHERE NOT EXISTS (" .
       "SELECT * FROM `install_target_providers`" .
       " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
@@ -54,7 +62,10 @@
         print "<font color=\"#00ff00\">(marked as to-be-deleted) ";
       else
         print "<font color=\"#ff0000\">";
-      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package.<br>";
+      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package";
+      if (isset($row["subst_repository"]))
+        print " - but can be replaced by the one in " . $row["subst_repository"];
+      print ".<br>";
       print "</font>\n";
     }
 
-- 
cgit v1.2.3-54-g00ecf


From fe270739334afaa072b7e7b43f9828a527250108 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 17 Mar 2018 18:46:45 +0100
Subject: bin/build-master-status-from-mysql: deep42thought -> girls

---
 bin/build-master-status-from-mysql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index a72d396..3918489 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -248,7 +248,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
 
 if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
   [ ! -s "${work_dir}/build-master-sanity" ]; then
-  printf 'deep42thought: my database is dirty again ...\n' | \
+  printf 'girls: my database is dirty again ...\n' | \
     sponge "${irc_dir}/#archlinux-ports/in"
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
-- 
cgit v1.2.3-54-g00ecf


From 5050d2272c44a1de655f914aa0181135d7de6f45 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 17 Mar 2018 20:14:35 +0100
Subject: web-scripts/mysql-issues.php: show if a package is already
 rescheduled

---
 web-scripts/mysql-issues.php | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/web-scripts/mysql-issues.php b/web-scripts/mysql-issues.php
index 485bc21..2f1d99d 100755
--- a/web-scripts/mysql-issues.php
+++ b/web-scripts/mysql-issues.php
@@ -28,7 +28,8 @@
     "`architectures`.`name`) AS `pkgfile`," .
     "`install_targets`.`name` AS `install_target`," .
     "IF(`binary_packages`.`is_to_be_deleted`,1,0) AS `is_to_be_deleted`," .
-    "`subst_r`.`name` AS `subst_repository`" .
+    "`subst_r`.`name` AS `subst_repository`," .
+    "`subst_buildlist_bp`.`id` AS `subst_buildlist`" .
     " FROM `binary_packages`" .
     " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
     " AND `repositories`.`is_on_master_mirror`" .
@@ -44,6 +45,11 @@
     " ON `subst_bp`.`pkgname`=`binary_packages`.`pkgname`" .
     " AND `subst_bp`.`id`!=`binary_packages`.`id`" .
     " AND `repository_stability_relations`.`more_stable`=`repositories`.`id`" .
+    " LEFT JOIN (`binary_packages` AS `subst_buildlist_bp`" .
+    " JOIN `repositories` AS `subst_buildlist_r`" .
+    " ON `subst_buildlist_bp`.`repository`=`subst_buildlist_r`.`id`" .
+    " AND `subst_buildlist_r`.`name`=\"build-list\"".
+    ") ON `subst_buildlist_bp`.`pkgname`=`binary_packages`.`pkgname`" .
     " WHERE NOT EXISTS (" .
       "SELECT * FROM `install_target_providers`" .
       " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
@@ -65,6 +71,8 @@
       print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package";
       if (isset($row["subst_repository"]))
         print " - but can be replaced by the one in " . $row["subst_repository"];
+      elseif (isset($row["subst_buildlist"]))
+        print " - but is already rescheduled";
       print ".<br>";
       print "</font>\n";
     }
-- 
cgit v1.2.3-54-g00ecf


From 8d6abf54b807e9e1132f7b6dddff0fdf0128aa7c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sun, 18 Mar 2018 13:30:37 +0100
Subject: lib/common-functions: remove done todo

---
 lib/common-functions | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/common-functions b/lib/common-functions
index c7a64fe..1abe7bc 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -4,8 +4,6 @@
 
 # shellcheck disable=SC2039
 
-# TODO: include link depenendencies in run-depends metadata
-
 # TODO: have full information (currently in files) in database
 
 # TODO: remove state files / metadata files
-- 
cgit v1.2.3-54-g00ecf


From 8782db8f68faf285da31ec7c21c1adae9a42f753 Mon Sep 17 00:00:00 2001
From: deep-42-thought <github@eckner.net>
Date: Mon, 19 Mar 2018 19:41:31 +0100
Subject: README.md: we need a mysqld

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f98b23e..820e73b 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ This includes scripts to be run on the build master as well as scripts to be run
 ### build master only
 * some ssh-server
 * `git`
-* `pkgbuild-introspection`
+* some mysql-server
 * `bc`
 ### build slave only
 * some ssh-client
-- 
cgit v1.2.3-54-g00ecf


From 39771d76fb8b5c85947cddfb0bb500353ecf6687 Mon Sep 17 00:00:00 2001
From: deep-42-thought <github@eckner.net>
Date: Mon, 19 Mar 2018 21:59:30 +0100
Subject: README.md: we need "screen"

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 820e73b..a389aa4 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,7 @@ This includes scripts to be run on the build master as well as scripts to be run
 ### build slave only
 * some ssh-client
 * `wget`
+* `screen`
 * `sudo` rights for `staging-with-build-support-i686-build`, `staging-i686-build`, `multilib-build`, `extra-x86_64-build`
 * `devtools32`
 
-- 
cgit v1.2.3-54-g00ecf


From 874a019c255794194fe68e8649516a3964f01af0 Mon Sep 17 00:00:00 2001
From: deep-42-thought <github@eckner.net>
Date: Mon, 19 Mar 2018 22:00:22 +0100
Subject: README.md: the _master_ needs "screen", not the slave

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a389aa4..64d57d4 100644
--- a/README.md
+++ b/README.md
@@ -8,11 +8,11 @@ This includes scripts to be run on the build master as well as scripts to be run
 * some ssh-server
 * `git`
 * some mysql-server
+* `screen`
 * `bc`
 ### build slave only
 * some ssh-client
 * `wget`
-* `screen`
 * `sudo` rights for `staging-with-build-support-i686-build`, `staging-i686-build`, `multilib-build`, `extra-x86_64-build`
 * `devtools32`
 
-- 
cgit v1.2.3-54-g00ecf


From 4b86fa14c4d28b83e026a1bb8fb571922c3c8815 Mon Sep 17 00:00:00 2001
From: deep-42-thought <github@eckner.net>
Date: Mon, 19 Mar 2018 22:03:42 +0100
Subject: README.md: we need "ii"

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 64d57d4..27b81f3 100644
--- a/README.md
+++ b/README.md
@@ -5,11 +5,12 @@ This includes scripts to be run on the build master as well as scripts to be run
 ## requirements
 * `moreutils`
 ### build master only
-* some ssh-server
+* `bc`
 * `git`
+* `ii`
 * some mysql-server
 * `screen`
-* `bc`
+* some ssh-server
 ### build slave only
 * some ssh-client
 * `wget`
-- 
cgit v1.2.3-54-g00ecf


From 6a69a4c9f2b38b10c26202847f8e64e47a0306fe Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 19 Mar 2018 22:10:52 +0100
Subject: #archlinux-ports -> #archlinux32

---
 bin/build-master-status-from-mysql | 2 +-
 bin/get-package-updates            | 2 +-
 bin/ii-connect                     | 6 +++---
 bin/return-assignment              | 4 ++--
 bin/sanity-check                   | 4 ++--
 lib/mysql-functions                | 2 +-
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 3918489..2c33708 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -249,7 +249,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
 if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
   [ ! -s "${work_dir}/build-master-sanity" ]; then
   printf 'girls: my database is dirty again ...\n' | \
-    sponge "${irc_dir}/#archlinux-ports/in"
+    sponge "${irc_dir}/#archlinux32/in"
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
 fi
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1cf8782..d3cb140 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -480,7 +480,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then
       done
     } | \
       tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
-      sponge "${irc_dir}/#archlinux-ports/in"
+      sponge "${irc_dir}/#archlinux32/in"
   fi
   exit 3
 fi
diff --git a/bin/ii-connect b/bin/ii-connect
index 973b227..cf332c0 100755
--- a/bin/ii-connect
+++ b/bin/ii-connect
@@ -19,12 +19,12 @@ if tail -n1 "${irc_dir}/nickserv/out" 2> /dev/null | \
     sponge "${irc_dir}/nickserv/in"
 fi
 
-# join #archlinux-ports if not yet done
+# join #archlinux32 if not yet done
 if ! grep ' buildmaster\.archlinux32\.org .* buildmaster$' "${irc_dir}/out" | \
   tail -n1 | \
-  grep -q ' #archlinux-ports '; then
+  grep -q ' #archlinux32 '; then
   {
-    echo '/j #archlinux-ports'
+    echo '/j #archlinux32'
     echo '/WHO buildmaster'
   } | \
     sponge "${irc_dir}/in"
diff --git a/bin/return-assignment b/bin/return-assignment
index a61460e..417654b 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -222,7 +222,7 @@ if [ "$5" = 'ERROR' ]; then
       fi
     )
 
-    if [ -p "${irc_dir}/#archlinux-ports/in" ]; then
+    if [ -p "${irc_dir}/#archlinux32/in" ]; then
       {
         printf '%s is broken (says %s).' \
           "$1" \
@@ -235,7 +235,7 @@ if [ "$5" = 'ERROR' ]; then
         fi
         printf '\n'
       } | \
-        sponge "${irc_dir}/#archlinux-ports/in"
+        sponge "${irc_dir}/#archlinux32/in"
     fi
   fi
 
diff --git a/bin/sanity-check b/bin/sanity-check
index b3e02aa..d98332c 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -22,7 +22,7 @@ usage() {
 i_am_insane() {
   if [ ! -s "${work_dir}/build-master-sanity" ]; then
     printf '\001ACTION goes insane.\001\n' | \
-      sponge "${irc_dir}/#archlinux-ports/in"
+      sponge "${irc_dir}/#archlinux32/in"
   fi
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
@@ -432,5 +432,5 @@ done
 if [ -f "${work_dir}/build-master-sanity" ]; then
   rm "${work_dir}/build-master-sanity"
   printf '\001ACTION resumes sanity.\001\n' | \
-    sponge "${irc_dir}/#archlinux-ports/in"
+    sponge "${irc_dir}/#archlinux32/in"
 fi
diff --git a/lib/mysql-functions b/lib/mysql-functions
index b9ae4ab..0b21bdf 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -83,7 +83,7 @@ mysql_run_query() {
       if [ ! -s "${work_dir}/build-master-sanity" ]; then
         printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \
           "${query_file##*/}" \
-          | sponge "${irc_dir}/#archlinux-ports/in"
+          | sponge "${irc_dir}/#archlinux32/in"
       fi
       echo 'A mysql query failed.' > \
         "${work_dir}/build-master-sanity"
-- 
cgit v1.2.3-54-g00ecf


From 6895ed0cb771fb5f489320b42f5148b70413c481 Mon Sep 17 00:00:00 2001
From: deep-42-thought <github@eckner.net>
Date: Mon, 19 Mar 2018 22:19:02 +0100
Subject: README.md: we need rsync on the buildmaster

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 27b81f3..3838928 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@ This includes scripts to be run on the build master as well as scripts to be run
 * `git`
 * `ii`
 * some mysql-server
+* `rsync`
 * `screen`
 * some ssh-server
 ### build slave only
-- 
cgit v1.2.3-54-g00ecf


From d0ad91d00c998a86db63edac9de68cefd405dc18 Mon Sep 17 00:00:00 2001
From: deep-42-thought <github@eckner.net>
Date: Tue, 20 Mar 2018 00:23:52 +0100
Subject: README.md: the buildmaster needs base-devel

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 3838928..cfbc032 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,7 @@ This includes scripts to be run on the build master as well as scripts to be run
 ## requirements
 * `moreutils`
 ### build master only
+* `base-devel`
 * `bc`
 * `git`
 * `ii`
-- 
cgit v1.2.3-54-g00ecf


From 6726701a1ad103bf3894d77ce062d0b849244388 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 06:35:51 +0100
Subject: bin/build-master-status-from-mysql: make message to irc more
 ambiguous

---
 bin/build-master-status-from-mysql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 2c33708..6ad9596 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -248,7 +248,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
 
 if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
   [ ! -s "${work_dir}/build-master-sanity" ]; then
-  printf 'girls: my database is dirty again ...\n' | \
+  printf 'oh girls, my database is dirty again ...\n' | \
     sponge "${irc_dir}/#archlinux32/in"
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
-- 
cgit v1.2.3-54-g00ecf


From 09a1fb32f0480e8493fcd7e9560a73044f784355 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 09:27:41 +0100
Subject: README.md: add more dependencies for buildmaster

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index cfbc032..985bc5c 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,9 @@ This includes scripts to be run on the build master as well as scripts to be run
 * `git`
 * `ii`
 * some mysql-server
+* `nginx` or equivalent
+* `php-gd`
+* `php-fpm` or quivalent
 * `rsync`
 * `screen`
 * some ssh-server
-- 
cgit v1.2.3-54-g00ecf


From 816a6c66f443c8dfdb3892e3c77b76cbbf7125ab Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 10:06:47 +0100
Subject: bin/bootstrap-mysql: table `statistics` new

---
 bin/bootstrap-mysql | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index bd6df9c..4c1938f 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -143,7 +143,22 @@ if [ ! "$1" = 'slim' ]; then
       ' from_repository MEDIUMINT :repositories' \
       ' to_repository MEDIUMINT :repositories' \
       ' upstream_package_repository SMALLINT :upstream_repositories' \
-      ' UNIQUE source from_repository upstream_package_repository'
+      ' UNIQUE source from_repository upstream_package_repository' \
+      'statistics BIGINT' \
+      ' date DATETIME' \
+      ' stable_packages_count MEDIUMINT' \
+      ' pending_tasks_count MEDIUMINT' \
+      ' pending_packages_count MEDIUMINT' \
+      ' staging_packages_count MEDIUMINT' \
+      ' testing_packages_count MEDIUMINT' \
+      ' tested_packages_count MEDIUMINT' \
+      ' broken_tasks_count MEDIUMINT' \
+      ' dependency_loops_count MEDIUMINT' \
+      ' dependency_looped_tasks_count MEDIUMINT' \
+      ' locked_tasks_count MEDIUMINT' \
+      ' blocked_tasks_count MEDIUMINT' \
+      ' next_tasks_count MEDIUMINT' \
+      ' UNIQUE date'
   )
 
   # shellcheck disable=SC2016
-- 
cgit v1.2.3-54-g00ecf


From abfd5ef7e34ed5e5f6e4ab46665d72b3f52ae0ec Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 10:11:44 +0100
Subject: bin/build-master-status-from-mysql: fix comment

---
 bin/build-master-status-from-mysql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 6ad9596..a444010 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -10,7 +10,7 @@
 tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-# always block if locked
+# do not block if locked
 exec 9> "${sanity_check_lock_file}"
 if ! flock -n 9; then
   >&2 echo 'Mysql-Sanity check skipped, cannot acquire lock.'
-- 
cgit v1.2.3-54-g00ecf


From 90a9a1999e0d5a82dca830d43f9e73fd87392dd7 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 10:26:19 +0100
Subject: bin/build-master-status-from-mysql: keep statistics in mysql database

---
 bin/build-master-status-from-mysql | 107 +++++++++++++++++++++++++++++++++++++
 1 file changed, 107 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index a444010..729375f 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -253,3 +253,110 @@ if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
 fi
+
+if [ ! -s "${work_dir}/build-master-sanity" ]; then
+  # shellcheck disable=SC2016
+  {
+    printf 'INSERT IGNORE INTO `statistics` ('
+    printf '`%s`,' \
+      'date' \
+      'stable_packages_count' \
+      'pending_tasks_count' \
+      'pending_packages_count' \
+      'staging_packages_count' \
+      'testing_packages_count' \
+      'tested_packages_count' \
+      'broken_tasks_count' \
+      'dependency_loops_count' \
+      'dependency_looped_tasks_count' \
+      'locked_tasks_count' \
+      'blocked_tasks_count' \
+      'next_tasks_count' | \
+      sed 's/,$//'
+    printf ') VALUES ('
+    # date
+    printf 'NOW(),'
+    # stable_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="stable"),'
+    # pending_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"),'
+    # pending_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"),'
+    # staging_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="staging"),'
+    # testing_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="testing"'
+    printf ' AND NOT `binary_packages`.`is_tested`),'
+    # tested_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="testing"'
+    printf ' AND `binary_packages`.`is_tested`),'
+    # broken_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_assignments`.`is_broken`),'
+    # dependency_loops_count
+    printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM'
+    printf ' `build_dependency_loops`),'
+    # dependency_looped_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM'
+    printf ' `build_dependency_loops`),'
+    # locked_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM'
+    printf ' `build_slaves`'
+    mysql_join_build_slaves_build_assignments
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"),'
+    # blocked_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_assignments`.`is_blocked` IS NOT NULL),'
+    # next_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND NOT EXISTS ('
+      printf 'SELECT * FROM `dependencies`'
+      mysql_join_dependencies_dependency_types
+      printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+      mysql_join_dependencies_install_target_providers
+      mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+      mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+      printf ' WHERE `prov_r`.`name`="build-list"'
+      printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`'
+    printf '))'
+    printf ');\n'
+  } | \
+    mysql_run_query
+fi
-- 
cgit v1.2.3-54-g00ecf


From bd5173dc0c845e0e01de8aed01bd85b0e6646344 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 10:33:35 +0100
Subject: lib/common-functions: irc_say new

---
 bin/build-master-status-from-mysql |  2 +-
 bin/get-package-updates            |  2 +-
 bin/return-assignment              | 28 +++++++++++++---------------
 bin/sanity-check                   |  4 ++--
 lib/common-functions               |  8 ++++++++
 lib/mysql-functions                |  2 +-
 6 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 729375f..e4dd313 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -249,7 +249,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
 if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
   [ ! -s "${work_dir}/build-master-sanity" ]; then
   printf 'oh girls, my database is dirty again ...\n' | \
-    sponge "${irc_dir}/#archlinux32/in"
+    irc_say
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
 fi
diff --git a/bin/get-package-updates b/bin/get-package-updates
index d3cb140..a6c3f32 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -480,7 +480,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then
       done
     } | \
       tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
-      sponge "${irc_dir}/#archlinux32/in"
+      irc_say
   fi
   exit 3
 fi
diff --git a/bin/return-assignment b/bin/return-assignment
index 417654b..c259f1c 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -222,21 +222,19 @@ if [ "$5" = 'ERROR' ]; then
       fi
     )
 
-    if [ -p "${irc_dir}/#archlinux32/in" ]; then
-      {
-        printf '%s is broken (says %s).' \
-          "$1" \
-          "${slave}"
-        if [ -n "${rescheduled_packages}" ]; then
-          printf -- ' - I rescheduled:'
-          # shellcheck disable=SC2086
-          printf ' %s,' ${rescheduled_packages} | \
-            sed 's/,$/./'
-        fi
-        printf '\n'
-      } | \
-        sponge "${irc_dir}/#archlinux32/in"
-    fi
+    {
+      printf '%s is broken (says %s).' \
+        "$1" \
+        "${slave}"
+      if [ -n "${rescheduled_packages}" ]; then
+        printf -- ' - I rescheduled:'
+        # shellcheck disable=SC2086
+        printf ' %s,' ${rescheduled_packages} | \
+          sed 's/,$/./'
+      fi
+      printf '\n'
+    } | \
+      irc_say
   fi
 
   exit 0
diff --git a/bin/sanity-check b/bin/sanity-check
index d98332c..9615ac7 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -22,7 +22,7 @@ usage() {
 i_am_insane() {
   if [ ! -s "${work_dir}/build-master-sanity" ]; then
     printf '\001ACTION goes insane.\001\n' | \
-      sponge "${irc_dir}/#archlinux32/in"
+      irc_say
   fi
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
@@ -432,5 +432,5 @@ done
 if [ -f "${work_dir}/build-master-sanity" ]; then
   rm "${work_dir}/build-master-sanity"
   printf '\001ACTION resumes sanity.\001\n' | \
-    sponge "${irc_dir}/#archlinux32/in"
+    irc_say
 fi
diff --git a/lib/common-functions b/lib/common-functions
index 1abe7bc..aa32409 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -1390,3 +1390,11 @@ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() {
     epoch="${epoch%%:*}"
   fi
 }
+
+# irc_say
+# say content of stdin in irc channel
+irc_say() {
+  if [ -p "${irc_dir}/#archlinux32/in" ]; then
+    sponge "${irc_dir}/#archlinux32/in"
+  fi
+}
diff --git a/lib/mysql-functions b/lib/mysql-functions
index 0b21bdf..801eca2 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -83,7 +83,7 @@ mysql_run_query() {
       if [ ! -s "${work_dir}/build-master-sanity" ]; then
         printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \
           "${query_file##*/}" \
-          | sponge "${irc_dir}/#archlinux32/in"
+          | irc_say
       fi
       echo 'A mysql query failed.' > \
         "${work_dir}/build-master-sanity"
-- 
cgit v1.2.3-54-g00ecf


From 85201f7de2d131aba3c4e10852aed3e11f39fa0a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 11:28:28 +0100
Subject: web-scripts/statistics.php: take info from database

---
 web-scripts/statistics.php | 100 ++++++++++++++++++++-------------------------
 1 file changed, 45 insertions(+), 55 deletions(-)

diff --git a/web-scripts/statistics.php b/web-scripts/statistics.php
index 279aa39..1e4d89b 100644
--- a/web-scripts/statistics.php
+++ b/web-scripts/statistics.php
@@ -1,54 +1,44 @@
 <?php
 
-$columns = array(
-  'stable',
-  'tasks',
-  'pending_packages',
-  'staging',
-  'testing',
-  'broken',
-  'loops',
-  'looped_packages',
-  'locked',
-  'blocked',
-  'next_pending',
-  'tested'
-);
-
-$print_columns = array(
-  'tasks',
-  'pending_packages',
-  'staging',
-  'testing',
-  'tested',
-  'broken',
-  'loops',
-  'looped_packages',
-  'locked',
-  'blocked',
-  'next_pending'
-);
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+if (! $result = $mysql -> query(
+    "SELECT DISTINCT ".
+    "UNIX_TIMESTAMP(`statistics`.`date`) AS `date`," .
+    "`statistics`.`pending_tasks_count`," .
+    "`statistics`.`pending_packages_count`," .
+    "`statistics`.`staging_packages_count`," .
+    "`statistics`.`testing_packages_count`," .
+    "`statistics`.`tested_packages_count`," .
+    "`statistics`.`broken_tasks_count`," .
+    "`statistics`.`dependency_loops_count`," .
+    "`statistics`.`dependency_looped_tasks_count`," .
+    "`statistics`.`locked_tasks_count`," .
+    "`statistics`.`blocked_tasks_count`," .
+    "`statistics`.`next_tasks_count`" .
+    "FROM `statistics` " .
+    "WHERE `statistics`.`date`>=ADDTIME(NOW(),\"-7 00:00:00\") " .
+    "ORDER BY `statistics`.`date`"
+  ))
+  die($mysql->error);
 
 $t_min = -1;
 $t_max = -1;
 $val_max = -1;
 
-foreach (explode("\n",trim(file_get_contents('/srv/http/statistics'))) as $val) {
-  $val = explode(" ",$val);
-  if (($t_min == -1) || ($t_min > $val[0]))
-    $t_min = $val[0];
-  if (($t_max == -1) || ($t_max < $val[0]))
-    $t_max = $val[0];
-  foreach ($columns as $id => $column) {
-    if (count($val) > $id+1)
-      $values[$column][$val[0]] = $val[$id+1];
-  };
-  foreach ($print_columns as $column) {
-    if (array_key_exists($column,$values))
-      if (($val_max == -1) || ($val_max < $values[$column][$val[0]]))
-        $val_max = $values[$column][$val[0]];
-  }
+while($vals = $result->fetch_assoc()) {
+  if ($t_min == -1)
+    $t_min = $vals["date"];
+  $t_max = $vals["date"];
+  foreach ($vals as $column => $val)
+    if ($column != "date") {
+      $values[$column][$vals["date"]] = $val;
+      $val_max = max($val_max,$val);
+    }
 };
+$print_columns = array_keys($values);
 
 $max_len = 0;
 foreach ($print_columns as $column) {
@@ -69,18 +59,18 @@ $im = @ImageCreate ($width + $legend_line_length + $max_len * ImageFontWidth(5),
 $background_color = ImageColorAllocate ($im, 255, 255, 255);
 $foreground_color = ImageColorAllocate ($im, 0, 0, 0);
 
-$colors['stable'] = ImageColorAllocate ($im, 0, 0, 0);
-$colors['tasks'] = ImageColorAllocate ($im, 0, 0, 128);
-$colors['pending_packages'] = ImageColorAllocate ($im, 0, 0, 255);
-$colors['staging'] = ImageColorAllocate ($im, 0, 100, 0);
-$colors['testing'] = ImageColorAllocate ($im, 0, 200, 0);
-$colors['tested'] = ImageColorAllocate ($im, 100, 255, 0);
-$colors['broken'] = ImageColorAllocate ($im, 255, 0, 0);
-$colors['loops'] = ImageColorAllocate ($im, 128, 128, 0);
-$colors['looped_packages'] = ImageColorAllocate ($im, 255, 128, 128);
-$colors['locked'] = ImageColorAllocate ($im, 128, 128, 128);
-$colors['blocked'] = ImageColorAllocate ($im, 128, 0, 0);
-$colors['next_pending'] = ImageColorAllocate ($im, 0, 255, 255);
+$colors['stable_packages_count'] = ImageColorAllocate ($im, 0, 0, 0);
+$colors['pending_tasks_count'] = ImageColorAllocate ($im, 0, 0, 128);
+$colors['pending_packages_count'] = ImageColorAllocate ($im, 0, 0, 255);
+$colors['staging_packages_count'] = ImageColorAllocate ($im, 0, 100, 0);
+$colors['testing_packages_count'] = ImageColorAllocate ($im, 0, 200, 0);
+$colors['tested_packages_count'] = ImageColorAllocate ($im, 100, 255, 0);
+$colors['broken_tasks_count'] = ImageColorAllocate ($im, 255, 0, 0);
+$colors['dependency_loops_count'] = ImageColorAllocate ($im, 128, 128, 0);
+$colors['dependency_looped_tasks_count'] = ImageColorAllocate ($im, 255, 128, 128);
+$colors['locked_tasks_count'] = ImageColorAllocate ($im, 128, 128, 128);
+$colors['blocked_tasks_count'] = ImageColorAllocate ($im, 128, 0, 0);
+$colors['next_tasks_count'] = ImageColorAllocate ($im, 0, 255, 255);
 
 function scale($x, $x_min, $x_max, $scale, $log) {
   if ($log) {
-- 
cgit v1.2.3-54-g00ecf


From e955c7d36c794056fd777398ac37c466cde8ddc3 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 11:52:06 +0100
Subject: web-scripts/statistics.php: make legend nice(r) again

---
 web-scripts/statistics.php | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/web-scripts/statistics.php b/web-scripts/statistics.php
index 1e4d89b..41b205f 100644
--- a/web-scripts/statistics.php
+++ b/web-scripts/statistics.php
@@ -51,7 +51,7 @@ $width = 1600;
 $height = 600;
 $border = 5;
 $legend_line_length = 10;
-$legend_height = 3 * ImageFontHeight(5) + $legend_line_length;
+$legend_height = 4 * ImageFontHeight(5) + $legend_line_length;
 
 $im = @ImageCreate ($width + $legend_line_length + $max_len * ImageFontWidth(5), $height + $legend_height)
       or die ("Cannot create new gd-image-stream");
@@ -118,13 +118,13 @@ function print_graph($data, $color) {
 
 ImageRectangle($im, $legend_line_length, 0, $width-1+$legend_line_length, $height-1, $foreground_color);
 
-ImageString($im, 5, $legend_line_length, $height + $legend_line_length + 2*ImageFontHeight(5), "( ".trim(shell_exec("uptime | sed 's|^.*\\s\\(load\\)|\\1|'"))." )", $foreground_color);
+ImageString($im, 5, $legend_line_length, $height + 2*$legend_line_length + 2*ImageFontHeight(5), "( ".trim(shell_exec("uptime | sed 's|^.*\\s\\(load\\)|\\1|'"))." )", $foreground_color);
 
 $xpos = $legend_line_length;
 foreach ($print_columns as $column) {
   print_graph($values[$column], $colors[$column]);
-  ImageString($im, 5, $xpos, $height + $legend_line_length + ImageFontHeight(5), $column, $colors[$column]);
-  $xpos += (strlen($column) + 1.75) * ImageFontWidth(5);
+  ImageString($im, 5, $xpos, $height + $legend_line_length + ImageFontHeight(5), substr($column,0,-strlen("_count")), $colors[$column]);
+  $xpos += (strlen($column) - strlen("_count") + 1.75) * ImageFontWidth(5);
 }
 
 ImageString($im, 5, $legend_line_length, $height + $legend_line_length, date('Y-m-d H:i', $t_min), $foreground_color);
-- 
cgit v1.2.3-54-g00ecf


From 301b04263a7d6343704b35bfd68961d536772daf Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 12:02:57 +0100
Subject: web-scripts/statistics.php: remove obsolete comment

---
 web-scripts/statistics.php | 2 --
 1 file changed, 2 deletions(-)

diff --git a/web-scripts/statistics.php b/web-scripts/statistics.php
index 41b205f..01bbb52 100644
--- a/web-scripts/statistics.php
+++ b/web-scripts/statistics.php
@@ -164,6 +164,4 @@ header ("Content-type: image/png");
 
 ImagePNG ($im);
 
-// passthru('wc -l /srv/http/statistics');
-
 ?>
-- 
cgit v1.2.3-54-g00ecf


From 31b97c033f957bbd0cd3f63eef1dc38e7e4eb5d6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 12:19:57 +0100
Subject: bin/build-master-status: cleanup

---
 bin/build-master-status | 870 +++++++++++-------------------------------------
 1 file changed, 193 insertions(+), 677 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index aa15fd4..9e3f377 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -12,32 +12,24 @@ usage() {
   >&2 echo 'build-master-status: report about status of build master'
   >&2 echo ''
   >&2 echo 'possible options:'
-  >&2 echo '  -w|--web:'
-  >&2 echo '    Output to webserver instead of stdout.'
   >&2 echo '  -h|--help:'
   >&2 echo '    Show this help and exit.'
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
 eval set -- "$(
-  getopt -o hw \
+  getopt -o h \
     --long help \
-    --long web \
     -n "$(basename "$0")" -- "$@" || \
   echo usage
 )"
 
-web=false
-
 while true
 do
   case "$1" in
     -h|--help)
       usage 0
     ;;
-    -w|--web)
-      web=true
-    ;;
     --)
       shift
       break
@@ -63,691 +55,215 @@ fi
 tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-stable=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="stable";\n'
-  } | \
-    mysql_run_query
-)
-tasks=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list";\n'
-  } | \
-    mysql_run_query
-)
-pending_packages=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list";\n'
-  } | \
-    mysql_run_query
-)
-next_tasks=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"'
-    printf ' AND NOT EXISTS ('
-      printf 'SELECT * FROM `dependencies`'
-      mysql_join_dependencies_dependency_types
-      printf ' AND `dependency_types`.`relevant_for_binary_packages`'
-      mysql_join_dependencies_install_target_providers
-      mysql_join_install_target_providers_binary_packages '' 'prov_bp'
-      mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
-      printf ' WHERE `prov_r`.`name`="build-list"'
-      printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`'
-    printf ')'
-    printf ';\n'
-  } | \
-    mysql_run_query
-)
-staging=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="staging";\n'
-  } | \
-    mysql_run_query
-)
-testing=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="testing"'
-    printf ' AND NOT `binary_packages`.`is_tested`;\n'
-  } | \
-    mysql_run_query
-)
-tested=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="testing"'
-    printf ' AND `binary_packages`.`is_tested`;\n'
-  } | \
-    mysql_run_query
-)
-{
-  find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
-    sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
-    while read -r pkg rev mod_rev repo; do
-      if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then
-        echo "${pkg}"
-      fi
-    done
-  {
-    find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-      -regex '.*/loop_[0-9]\+' \
-      -exec cat '{}' \; | \
-      sort -u
-    find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
-      sed 's|\(\.[^.]\+\)\{4\}||' | \
-      sort -u
-  } | \
-    sort | \
-    uniq -d
-} | \
-  sort -u > \
-  "${tmp_dir}/broken-packages-names"
-broken=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"'
-    printf ' AND `build_assignments`.`is_broken`;\n'
-  } | \
-    mysql_run_query
-)
-blocked=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"'
-    printf ' AND `build_assignments`.`is_blocked` IS NOT NULL;\n'
-  } | \
-    mysql_run_query
-)
-locked=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM'
-    printf ' `build_slaves`'
-    mysql_join_build_slaves_build_assignments
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list";\n'
-  } | \
-    mysql_run_query
-)
-loops=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM'
-    printf ' `build_dependency_loops`'
-  } | \
-    mysql_run_query
-)
-looped_packages=$(
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM'
-    printf ' `build_dependency_loops`'
-  } | \
-    mysql_run_query
-)
+"${base_dir}/bin/calculate-dependent-packages"
 
 {
-  printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \
-    "${stable}" \
-    "$((staging+testing+tested+pending_packages))"
-  printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \
-    "$((tasks-broken))" \
-    "${tasks}" \
-    "${pending_packages}" \
-    "${next_tasks}"
-  printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \
-    "$((testing+tested))" \
-    "${tested}" \
-    "${staging}"
-  printf 'There are %d broken package builds.\n' \
-    "${broken}"
-  if [ "${loops}" -ne 0 ]; then
-    printf 'There are %d loops containing %d package builds.\n' \
-      "${loops}" \
-      "${looped_packages}"
-  fi
-  if [ $((broken+testing+tested+staging)) -ne 0 ]; then
-    printf '%.1f%% of all packages are broken.\n' \
-      "$(
-        echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \
-          bc
-      )"
-  fi
-  if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then
-    printf '%.1f%% of the planned work has been done.\n' \
-      "$(
-        echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \
-          bc
-      )"
-  fi
-} > \
-  "${tmp_dir}/build-master-status.html"
-
-if ${web}; then
-  "${base_dir}/bin/calculate-dependent-packages"
-  {
-    printf '%s\n' \
-      '<html>' \
-      '<head>' \
-      '<title>Status of archlinux32 build master</title>' \
-      '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
-      '</head>' \
-      '<body>'
-    sed 's|$|<br>|' "${tmp_dir}/build-master-status.html"
-    printf '%s\n' \
-      '<br>' \
-      'currently building packages:<br>' \
-      '<table>'
-    printf '<tr>'
-    printf '<th>%s</th>' \
-      'since (UTC)' \
-      'pkgname' \
-      'git revision' \
-      'modification git revision' \
-      'package repository' \
-      'build slave'
-    printf '</tr>'
-    find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \
-      -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \
-      -execdir sed '
-        :a
-          $!{
-            N
-            s/\n/, /
-            ba
-          }
-      ' '{}' \; | \
-      sort -k1n,1 | \
-      sed '
-        s|^\S\+ ||
-        s|\.locked | |
-        s|\.\([^.]\+\)$| \1|
-        s|\.\([^.]\+\)$| \1|
-        s|\.\([^.]\+\)$| \1|
-      ' | \
-      while read -r date time pkg rev mod_rev repo slaves; do
-        printf '<tr>'
-        printf '<td>%s</td>' \
-          "${date} ${time}" \
-          "${pkg}" \
-          "<p style=\"font-size:8px\">${rev}</p>" \
-          "<p style=\"font-size:8px\">$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")</p>" \
-          "${repo}" \
-          "${slaves}"
-        printf '</tr>\n'
-      done
-    printf '%s\n' \
-      '</table>' \
-      '</body>' \
-      '</html>'
-  } | \
-    sponge "${tmp_dir}/build-master-status.html"
-  end=$(($(date +%s)-7*24*60*60))
-  {
-    [ -f "${webserver_directory}/statistics" ] && \
-      cat "${webserver_directory}/statistics"
-    printf '%s ' \
-      "$(date +%s)" \
-      "${stable}" \
-      "${tasks}" \
-      "${pending_packages}" \
-      "${staging}" \
-      "${testing}" \
-      "${broken}" \
-      "${loops}" \
-      "${looped_packages}" \
-      "${locked}" \
-      "${blocked}" \
-      "${next_tasks}" \
-      "${tested}" | \
-      sed 's| $|\n|'
-    echo "${end}"
-  } | \
-    sort -k1nr,1 | \
-    sed -n "
-      /^${end}\$/q
+  printf '%s\n' \
+    '<html>' \
+    '<head>' \
+    '<title>Todos in the build scripts</title>' \
+    '</head>' \
+    '<body>'
+  find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
+    -exec grep -nHF '' '{}' \; | \
+    awk '
+      { print $0 }
+      /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i}
+    ' | \
+    sed -n '
+      s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/
+      T
+      N
+      s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/
+      :a
+        N
+        s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/
+        ta
+      s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
       p
-    " | \
-    tac > \
-    "${tmp_dir}/statistics"
-
-  find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \
-    \( \
-      -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \
-      -printf '%f build()\n' \
-    \) -o \
-    \( \
-      -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \
-      -printf '%f check()\n' \
-    \) -o \
-    \( \
-      -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \
-      -printf '%f prepare()\n' \
-    \) -o \
-    \( \
-      -exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \
-      -printf '%f package()\n' \
-    \) -o \
-    \( \
-      -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \
-      -printf '%f source\n' \
-    \) -o \
-    \( \
-      -exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \
-      -printf '%f dependencies\n' \
-    \) -o \
-    \( \
-      -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \
-      -printf '%f package-cache\n' \
-    \) -o \
-    \( \
-      -exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \
-      -printf '%f run-as-root\n' \
-    \) -o \
-    -printf '%f unknown\n' \
-    \) | \
-    sed '
-      s|\(\.[^.]\+\)\{3\} | |
     ' | \
-    sort -u | \
+    tee "${tmp_dir}/todos" | \
     sed '
       :a
-        $!N
-        s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/
-      ta
-      P
-      D
+      N
+      /\n$/!ba
+      s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n|
     ' | \
-    sort -k1,1 > \
-    "${tmp_dir}/broken-packages.reason"
-
-  {
-    printf '%s\n' \
-      '<html>' \
-      '<head>' \
-      '<title>List of broken package builds</title>' \
-      '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
-      '</head>' \
-      '<body>' \
-      '<a href="build-logs/">build logs</a><br>' \
-      '<table>' \
-      '<tr>'
-    printf '<th>%s</th>' \
-      'package' \
-      'git revision' \
-      'modification git revision' \
-      'package repository' \
-      'compilations' \
-      'dependent' \
-      'build error' \
-      'blocked'
-    printf '</tr>\n'
-    find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
-      sed 's|\.broken$||' | \
-      sort -k1,1 | \
-      join -j 1 - "${tmp_dir}/broken-packages.reason" | \
-      sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \
-      while read -r sf pkg rev mod_rev repo build_error; do
-        if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then
-          printf '1 '
-        else
-          printf '0 '
-        fi
-        printf '%s ' \
-          "${pkg}" \
-          "${rev}" \
-          "${mod_rev}" \
-          "${repo}" \
-          "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \
-          "$(
-            # shellcheck disable=SC2010
-            ls -t "${webserver_directory}/build-logs/error" | \
-              grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$'
-          )" \
-          "$(
-            {
-              grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \
-                echo 'x &nbsp;'
-            } | \
-              cut -d' ' -f2
-          )" \
-          "${build_error}"
-        if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then
-          sed '
-            s|\s\(wait for \)|\n\1|g
-          ' "${work_dir}/package-states/${sf}.blocked" | \
-            while read -r blocked_reason; do
-              if echo "${blocked_reason}" | \
-                grep -q '^wait for '; then
-                printf 'wait for '
-                echo "${blocked_reason}" | \
-                  sed '
-                    s|^wait for ||
-                    s@\( and \| or \)@\n\1\n@
-                  ' | \
-                  while read -r reason; do
-                    if [ "FS#${reason#FS#}" = "${reason}" ]; then
-                      printf '<a href="https://bugs.archlinux.org/task/%s">%s</a>' \
-                        "${reason#FS#}" \
-                        "${reason}"
-                    elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then
-                      printf '<a href="https://bugs.archlinux32.org/index.php?do=details&task_id=%s">%s</a>' \
-                        "${reason#FS32#}" \
-                        "${reason}"
-                    elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then
-                      printf '<a href="graphs/%s.png">%s</a>' \
-                        "${reason}" \
-                        "${reason}"
-                    elif [ "${reason% *}" != "${reason}" ]; then
-                      printf '%s' \
-                        "${reason}"
-                    else
-                      printf '<font color="red">%s</font>' \
-                        "${reason}"
-                    fi
-                    if read -r operator; then
-                      printf ' %s ' "${operator}"
-                    fi
-                  done
-              else
-                echo "${blocked_reason}"
-              fi
-            done | \
-              tr '\n' ' '
-        else
-          printf '&nbsp;'
-        fi
-        printf '\n'
-      done | \
-      sort -k6n,6 | \
-      while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do
-        if [ "${buildable}" -eq 0 ]; then
-          left='('
-          right=')'
-        else
-          unset left
-          unset right
-        fi
-        printf '<tr>'
-        mod_rev=$(
-          modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
-        )
-        build_error=$(
-          echo "${build_error}" | \
-            sed 's|,|, |g'
-        )
-        printf '<td>%s</td>' \
-          '<a href="graphs/'"${pkg}"'.png">'"${left}${pkg}${right}"'</a>' \
-          "<p style=\"font-size:8px\">${rev}</p>" \
-          "<p style=\"font-size:8px\">${mod_rev}</p>" \
-          "${repo}" \
-          '<a href="build-logs/error/'"${log_file}"'">'"${count}"'</a>' \
-          "${dependent}" \
-          "${build_error}" \
-          "${reason}"
-        printf '</tr>\n'
-      done
-    printf '%s\n' \
-      '</table>' \
-      '</body>' \
-      '</html>'
-  } > \
-    "${tmp_dir}/broken-packages.html"
+    sed '
+      s|$|<br>|
+    '
+  printf '%s\n' \
+    '</body>' \
+    '</html>'
+} > \
+  "${tmp_dir}/todos.html"
 
-  rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason"
+if [ -s "${tmp_dir}/todos" ]; then
+  sed '
+    :a
+    N
+    /\n$/!ba
+    s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 |
+    s/\n$//
+    s/\n/\\n/g
+  ' -i "${tmp_dir}/todos"
+  while read -r file line desc; do
+    printf '%s %s %s\n' \
+      "$(printf '%s' "${file}" | base64 -w0)" \
+      "$(printf '%s' "${line}" | base64 -w0)" \
+      "$(printf '%s' "${desc}" | base64 -w0)"
+  done < \
+    "${tmp_dir}/todos" | \
+    sponge "${tmp_dir}/todos"
+  # update todos
+  # shellcheck disable=SC2016
+  while read -r file line desc; do
+    printf 'UPDATE IGNORE `todos`'
+    printf ' SET `todos`.`line`=from_base64("%s")' \
+      "${line}"
+    printf ' WHERE `todos`.`file`=from_base64("%s")' \
+      "${file}"
+    printf ' AND `todos`.`description`=from_base64("%s");\n' \
+      "${desc}"
 
+    printf 'UPDATE IGNORE `todos`'
+    printf ' SET `todos`.`description`=from_base64("%s")' \
+      "${desc}"
+    printf ' WHERE `todos`.`file`=from_base64("%s")' \
+      "${file}"
+    printf ' AND `todos`.`line`=from_base64("%s");\n' \
+      "${line}"
+  done < \
+    "${tmp_dir}/todos" | \
+    mysql_run_query
+  # insert unfound todos
+  # shellcheck disable=SC2016
   {
-    printf '%s\n' \
-      '<html>' \
-      '<head>' \
-      '<title>Todos in the build scripts</title>' \
-      '</head>' \
-      '<body>'
-    find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
-      -exec grep -nHF '' '{}' \; | \
-      awk '
-        { print $0 }
-        /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i}
-      ' | \
-      sed -n '
-        s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/
-        T
-        N
-        s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/
-        :a
-          N
-          s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/
-          ta
-        s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
-        p
-      ' | \
-      tee "${tmp_dir}/todos" | \
+    printf 'SHOW CREATE TABLE `todos`' | \
+      mysql_run_query | \
       sed '
-        :a
-        N
-        /\n$/!ba
-        s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n|
-      ' | \
-      sed '
-        s|$|<br>|
+        1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
       '
-    printf '%s\n' \
-      '</body>' \
-      '</html>'
-  } > \
-    "${tmp_dir}/todos.html"
-
-  if [ -s "${tmp_dir}/todos" ]; then
-    sed '
-      :a
-      N
-      /\n$/!ba
-      s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 |
-      s/\n$//
-      s/\n/\\n/g
-    ' -i "${tmp_dir}/todos"
-    while read -r file line desc; do
-      printf '%s %s %s\n' \
-        "$(printf '%s' "${file}" | base64 -w0)" \
-        "$(printf '%s' "${line}" | base64 -w0)" \
-        "$(printf '%s' "${desc}" | base64 -w0)"
-    done < \
-      "${tmp_dir}/todos" | \
-      sponge "${tmp_dir}/todos"
-    # update todos
-    # shellcheck disable=SC2016
+    printf ';\n'
+    printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES '
     while read -r file line desc; do
-      printf 'UPDATE IGNORE `todos`'
-      printf ' SET `todos`.`line`=from_base64("%s")' \
-        "${line}"
-      printf ' WHERE `todos`.`file`=from_base64("%s")' \
-        "${file}"
-      printf ' AND `todos`.`description`=from_base64("%s");\n' \
-        "${desc}"
-
-      printf 'UPDATE IGNORE `todos`'
-      printf ' SET `todos`.`description`=from_base64("%s")' \
-        "${desc}"
-      printf ' WHERE `todos`.`file`=from_base64("%s")' \
-        "${file}"
-      printf ' AND `todos`.`line`=from_base64("%s");\n' \
-        "${line}"
+      printf '('
+      printf 'from_base64("%s"),' \
+        "${file}" \
+        "${line}" \
+        "${desc}" | \
+        sed 's/,$/),/'
     done < \
       "${tmp_dir}/todos" | \
-      mysql_run_query
-    # insert unfound todos
-    # shellcheck disable=SC2016
-    {
-      printf 'SHOW CREATE TABLE `todos`' | \
-        mysql_run_query | \
-        sed '
-          1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
-        '
-      printf ';\n'
-      printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES '
-      while read -r file line desc; do
-        printf '('
-        printf 'from_base64("%s"),' \
-          "${file}" \
-          "${line}" \
-          "${desc}" | \
-          sed 's/,$/),/'
-      done < \
-        "${tmp_dir}/todos" | \
-        sed '
-          s/,$//
-        '
-      printf ';\n'
-      printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) '
-      printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` '
-      printf 'FROM `td` '
-      printf 'WHERE NOT EXISTS ('
-      printf 'SELECT * FROM `todos`'
-      printf ' AND `td`.`%s`=`todos`.`%s`' \
-        'file' 'file' \
-        'line' 'line' \
-        'description' 'description' | \
-        sed 's/^ AND / WHERE /'
-      printf ');\n'
-
-      printf 'DELETE FROM `todos` WHERE NOT EXISTS ('
-      printf 'SELECT * FROM `td`'
-      printf ' AND `td`.`%s`=`todos`.`%s`' \
-        'file' 'file' \
-        'line' 'line' \
-        'description' 'description' | \
-        sed 's/^ AND / WHERE /'
-      printf ');'
-      printf 'DROP TABLE `td`;\n'
-      printf 'DELETE FROM `todo_links` WHERE NOT EXISTS ('
-      printf 'SELECT * FROM `todos` '
-      printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`'
-      printf ') OR NOT EXISTS ('
-      printf 'SELECT * FROM `todos` '
-      printf 'WHERE `todos`.`id`=`todo_links`.`dependent`'
-      printf ');\n'
-    } | \
-      mysql_run_query
-    rm -f "${tmp_dir}/todos"
-  fi
-
-  {
-    printf '%s\n' \
-      '<html>' \
-      '<head>' \
-      '<title>Blacklisted packages</title>' \
-      '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
-      '</head>' \
-      '<body>' \
-      '<table>'
-    printf '<tr>'
-    printf '<th>%s</th>' \
-      'package' \
-      'reason'
-    printf '</tr>\n'
-    git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
-      tar -Ox | \
       sed '
-        s@FS#\([0-9]\+\)@<a href="https://bugs.archlinux.org/task/\1">\0</a>@
-        s@FS32#\([0-9]\+\)@<a href="https://bugs.archlinux32.org/index.php?do=details\&task_id=\1">\0</a>@
-        /.#/!s/$/#/
-        s|\(.\)#|\1</td><td>|
-        /^\s*#/{
-          s/^\s*#\s*//
-          s|\s*\(</td><td>\)|</font></s>\1|
-          s/^/<s><font color="#808080">/
-        }
-        s|^|<tr><td>|
-        s|$|</td></tr>|
+        s/,$//
       '
-    printf '%s\n' \
-      '</table>' \
-      '</body>' \
-      '</html>'
-  } > \
-    "${tmp_dir}/blacklist.html"
+    printf ';\n'
+    printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) '
+    printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` '
+    printf 'FROM `td` '
+    printf 'WHERE NOT EXISTS ('
+    printf 'SELECT * FROM `todos`'
+    printf ' AND `td`.`%s`=`todos`.`%s`' \
+      'file' 'file' \
+      'line' 'line' \
+      'description' 'description' | \
+      sed 's/^ AND / WHERE /'
+    printf ');\n'
 
-  {
-    printf '%s\n' \
-      '<html>' \
-      '<head>' \
-      '<title>log of ssh connections from build slaves</title>' \
-      '</head>' \
-      '<body>' \
-      '<table>'
-    printf '<tr>'
-    printf '<th>%s</th>' \
-      'time' \
-      'build slave' \
-      'command' \
-      'arguments'
-    printf '</tr>\n'
-    if [ -r "${work_dir}/ssh-log" ]; then
-      tac "${work_dir}/ssh-log" | \
-        while read -r date time slave command arguments; do
-          printf '<tr>'
-          printf '<td>%s</td>' \
-            "${date} ${time}" \
-            "${slave}" \
-            "${command}" \
-            "${arguments}"
-          printf '</tr>\n'
-        done
-    fi
-    printf '%s\n' \
-      '</table>' \
-      '</body>' \
-      '</html>'
-  } > \
-    "${tmp_dir}/ssh-log.html"
+    printf 'DELETE FROM `todos` WHERE NOT EXISTS ('
+    printf 'SELECT * FROM `td`'
+    printf ' AND `td`.`%s`=`todos`.`%s`' \
+      'file' 'file' \
+      'line' 'line' \
+      'description' 'description' | \
+      sed 's/^ AND / WHERE /'
+    printf ');'
+    printf 'DROP TABLE `td`;\n'
+    printf 'DELETE FROM `todo_links` WHERE NOT EXISTS ('
+    printf 'SELECT * FROM `todos` '
+    printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`'
+    printf ') OR NOT EXISTS ('
+    printf 'SELECT * FROM `todos` '
+    printf 'WHERE `todos`.`id`=`todo_links`.`dependent`'
+    printf ');\n'
+  } | \
+    mysql_run_query
+  rm -f "${tmp_dir}/todos"
+fi
 
-  find "${tmp_dir}" -maxdepth 1 -type f | \
-    while read -r file; do
-      cat "${file}" > \
-        "${webserver_directory}/${file##*/}"
-    done
+{
+  printf '%s\n' \
+    '<html>' \
+    '<head>' \
+    '<title>Blacklisted packages</title>' \
+    '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
+    '</head>' \
+    '<body>' \
+    '<table>'
+  printf '<tr>'
+  printf '<th>%s</th>' \
+    'package' \
+    'reason'
+  printf '</tr>\n'
+  git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
+    tar -Ox | \
+    sed '
+      s@FS#\([0-9]\+\)@<a href="https://bugs.archlinux.org/task/\1">\0</a>@
+      s@FS32#\([0-9]\+\)@<a href="https://bugs.archlinux32.org/index.php?do=details\&task_id=\1">\0</a>@
+      /.#/!s/$/#/
+      s|\(.\)#|\1</td><td>|
+      /^\s*#/{
+        s/^\s*#\s*//
+        s|\s*\(</td><td>\)|</font></s>\1|
+        s/^/<s><font color="#808080">/
+      }
+      s|^|<tr><td>|
+      s|$|</td></tr>|
+    '
+  printf '%s\n' \
+    '</table>' \
+    '</body>' \
+    '</html>'
+} > \
+  "${tmp_dir}/blacklist.html"
 
-else
-  cat "${tmp_dir}/build-master-status.html"
-fi
+{
+  printf '%s\n' \
+    '<html>' \
+    '<head>' \
+    '<title>log of ssh connections from build slaves</title>' \
+    '</head>' \
+    '<body>' \
+    '<table>'
+  printf '<tr>'
+  printf '<th>%s</th>' \
+    'time' \
+    'build slave' \
+    'command' \
+    'arguments'
+  printf '</tr>\n'
+  if [ -r "${work_dir}/ssh-log" ]; then
+    tac "${work_dir}/ssh-log" | \
+      while read -r date time slave command arguments; do
+        printf '<tr>'
+        printf '<td>%s</td>' \
+          "${date} ${time}" \
+          "${slave}" \
+          "${command}" \
+          "${arguments}"
+        printf '</tr>\n'
+      done
+  fi
+  printf '%s\n' \
+    '</table>' \
+    '</body>' \
+    '</html>'
+} > \
+  "${tmp_dir}/ssh-log.html"
+
+find "${tmp_dir}" -maxdepth 1 -type f | \
+  while read -r file; do
+    cat "${file}" > \
+      "${webserver_directory}/${file##*/}"
+  done
-- 
cgit v1.2.3-54-g00ecf


From 112da5617dba1c83e5c3cfc10092c24ebc652a5f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 12:23:44 +0100
Subject: bin/build-master-status: do not call calculate-dependent-packages -
 noone uses the results currently

---
 bin/build-master-status | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 9e3f377..18a5d5b 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -55,8 +55,6 @@ fi
 tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-"${base_dir}/bin/calculate-dependent-packages"
-
 {
   printf '%s\n' \
     '<html>' \
-- 
cgit v1.2.3-54-g00ecf


From 5ed904c0b9fae476e45621b2e21b61ea8391db2b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 13:23:21 +0100
Subject: bin/build-master-status-from-mysql: ignore duplicates of handed out
 assignments

---
 bin/build-master-status-from-mysql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index e4dd313..5f27d89 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -36,7 +36,7 @@ sort "${work_dir}/build-list" > \
 
 # shellcheck disable=SC2016
 {
-  printf 'SELECT '
+  printf 'SELECT DISTINCT '
   printf '`package_sources`.`%s`,' \
     'pkgbase' 'git_revision' 'mod_git_revision'
   printf '`upstream_repositories`.`name`'
@@ -248,7 +248,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
 
 if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
   [ ! -s "${work_dir}/build-master-sanity" ]; then
-  printf 'oh girls, my database is dirty again ...\n' | \
+  printf 'girls, my database is dirty again ...\n' | \
     irc_say
   echo 'build master is insane' > \
     "${work_dir}/build-master-sanity"
-- 
cgit v1.2.3-54-g00ecf


From 6b86012dc547b674b8a5c347850e6ec3698433ee Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 14:33:03 +0100
Subject: remove bin/calculate-dependent-packages - this needs to be rewritten
 using the database ... _if_ we still want that information

---
 bin/calculate-dependent-packages | 188 ---------------------------------------
 1 file changed, 188 deletions(-)
 delete mode 100755 bin/calculate-dependent-packages

diff --git a/bin/calculate-dependent-packages b/bin/calculate-dependent-packages
deleted file mode 100755
index 3208e61..0000000
--- a/bin/calculate-dependent-packages
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/bin/sh
-
-# shellcheck source=conf/default.conf
-. "${0%/*}/../conf/default.conf"
-
-# TODO: save information in database
-
-# TODO: read information from database
-
-if [ -s "${work_dir}/build-master-sanity" ]; then
-  >&2 echo 'Build master is not sane.'
-  exit
-fi
-
-tmp_dir=$(mktemp -d 'tmp.calculate-dependent-packages.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-sort -k1,1 -u "${work_dir}/build-list" > \
-  "${tmp_dir}/build-list"
-
-while read -r pkg rev mod_rev repo; do
-  generate_package_metadata "${pkg}" "${rev}" "${mod_rev}" "${repo}"
-done < \
-  "${tmp_dir}/build-list"
-
-mkdir "${tmp_dir}/loops"
-find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-  -regex '.*/loop_[0-9]\+' | \
-  while read -r loop; do
-    sort -u "${loop}" | \
-      join -j 1 - "${tmp_dir}/build-list" | \
-      tr ' ' '.' > \
-      "${tmp_dir}/loops/${loop##*/}"
-  done
-
-touch "${tmp_dir}/dependent-count"
-
-tr ' ' '.' < \
-  "${tmp_dir}/build-list" | \
-  sponge "${tmp_dir}/build-list"
-
-sums=''
-
-while [ -s "${tmp_dir}/build-list" ] && [ "${sums}" != "$(sha512sum "${tmp_dir}/dependent-count")" ]; do
-
-  sums=$(
-    sha512sum "${tmp_dir}/dependent-count"
-  )
-
-  {
-    sed 's|^|? |' "${tmp_dir}/build-list"
-    sed 'p' "${tmp_dir}/dependent-count"
-  } | \
-    sort -k2,2 | \
-    uniq -uf1 | \
-    cut -d' ' -f2 | \
-    sponge "${tmp_dir}/build-list"
-
-  sed '
-    s|^|'"${work_dir}"'/package-infos/|
-    s|$|.build-depends|
-  ' "${tmp_dir}/build-list" | \
-    xargs -r grep -hvx 'base\|base-devel' | \
-    sort -u > \
-    "${tmp_dir}/build-list.build-depends"
-
-  sed '
-    s|^\S\+ |'"${work_dir}"'/package-infos/|
-    s|$|.build-depends|
-  ' "${tmp_dir}/dependent-count" | \
-    xargs -r grep -Hvx 'base\|base-devel' | \
-    sed '
-      s|^.*/||
-      s|\.build-depends:| |
-    ' | \
-    sort -k2,2 > \
-    "${tmp_dir}/dependent-count.build-depends"
-
-  rm -f "${tmp_dir}/dependent-count.new"
-  touch "${tmp_dir}/dependent-count.new"
-
-  while read -r sf; do
-    if [ -n "$(
-        {
-          sort -u "${work_dir}/package-infos/${sf}.builds"
-          cat "${tmp_dir}/build-list.build-depends"
-        } | \
-          sort | \
-          uniq -d
-      )" ]; then
-      continue
-    fi
-
-    count="/$(
-      sort -u "${work_dir}/package-infos/${sf}.builds" | \
-        join -1 1 -2 2 -o 2.1 - "${tmp_dir}/dependent-count.build-depends" | \
-        sort -u | \
-        join -1 1 -2 2 -o 2.1,2.2 - "${tmp_dir}/dependent-count" | \
-        tr '/ ' '\n' | \
-        grep -vxF '' | \
-        sort -u | \
-        tr '\n' '/'
-    )"
-    printf '%s %s\n' \
-      "${count}" \
-      "${sf}" >> \
-      "${tmp_dir}/dependent-count.new"
-    find "${tmp_dir}/loops" -type f -maxdepth 1 \
-      -exec grep -qxF "${sf}" {} \; \
-      -exec rm {} \;
-  done < \
-    "${tmp_dir}/build-list"
-
-  find "${tmp_dir}/loops" -maxdepth 1 | \
-    while read -r loop; do
-      if [ ! -f "${loop}" ]; then
-        continue
-      fi
-      if [ -n "$(
-          {
-            sed '
-              s|^|'"${work_dir}"'/package-infos/|
-              s|$|.builds|
-            ' "${loop}" | \
-              xargs -r cat | \
-              sort -u
-            {
-              sed 'p' "${loop}"
-              cat "${tmp_dir}/build-list"
-            } | \
-              sort | \
-              uniq -u | \
-              sed '
-                s|^|'"${work_dir}"'/package-infos/|
-                s|$|.build-depends|
-              ' | \
-              xargs -r grep -hvx 'base\|base-devel' | \
-              sort -u
-          } | \
-            sort | \
-            uniq -d
-        )" ]; then
-        continue
-      fi
-
-      count="/$(
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.builds|
-        ' "${loop}" | \
-          xargs -r cat | \
-          sort -u | \
-        join -1 1 -2 2 -o 2.1 - "${tmp_dir}/dependent-count.build-depends" | \
-        sort -u | \
-        join -1 1 -2 2 -o 2.1,2.2 - "${tmp_dir}/dependent-count" | \
-        tr '/ ' '\n' | \
-        grep -vxF '' | \
-        sort -u | \
-        tr '\n' '/'
-      )"
-      while read -r sf; do
-        printf '%s %s\n' "${count}" "${sf}" >> \
-          "${tmp_dir}/dependent-count.new"
-        find "${tmp_dir}/loops" -maxdepth 1 -type f -not -name "${loop##*/}" \
-          -exec grep -qxF "${sf}" {} \; \
-          -exec rm {} \;
-      done < \
-        "${loop}"
-
-      rm "${loop}"
-    done
-
-  cat "${tmp_dir}/dependent-count" "${tmp_dir}/dependent-count.new" | \
-    sort -k2,2 | \
-    sponge "${tmp_dir}/dependent-count"
-
-done
-
-while read -r count pkg; do
-  count=$(
-    echo "${count}" | \
-      tr '/' '\n' | \
-      grep -cvxF ''
-  ) || true
-  printf '%s %s\n' "${pkg}" "${count}"
-done < \
-  "${tmp_dir}/dependent-count" | \
-  sponge "${work_dir}/dependent-count"
-- 
cgit v1.2.3-54-g00ecf


From b7f0199b15fb07d2532b12c627bf400783e7a115 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 14:35:12 +0100
Subject: work/build-list.loops decomissioned

---
 bin/build-master-status-from-mysql | 21 -------------------
 bin/get-assignment                 |  6 ------
 bin/get-package-updates            | 42 ++------------------------------------
 bin/return-assignment              | 18 ----------------
 conf/default.conf                  |  1 -
 5 files changed, 2 insertions(+), 86 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 5f27d89..edabc1e 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -72,23 +72,6 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin
 sort "${work_dir}/deletion-list" > \
   "${tmp_dir}/deletion-list.file"
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT `package_sources`.`pkgbase`'
-  printf ' FROM `package_sources`'
-  mysql_join_package_sources_build_assignments
-  mysql_join_build_assignments_build_dependency_loops
-  printf ';\n'
-} | \
-  mysql_run_query | \
-  sort > \
-  "${tmp_dir}/loops.mysql"
-find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \
-  -regex '.*/loop_[0-9]\+' \
-  -exec 'cat' '{}' \; | \
-  sort > \
-  "${tmp_dir}/loops.file"
-
 # shellcheck disable=SC2016
 {
   printf 'SELECT '
@@ -220,10 +203,6 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     "${tmp_dir}/deletion-list.file" \
     "${tmp_dir}/deletion-list.mysql" || \
     true
-  diff -u \
-    "${tmp_dir}/loops.file" \
-    "${tmp_dir}/loops.mysql" >&2 || \
-    true
   diff -u \
     "${tmp_dir}/tested.file" \
     "${tmp_dir}/tested.mysql" || \
diff --git a/bin/get-assignment b/bin/get-assignment
index 2df3059..3357fc6 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -103,12 +103,6 @@ hand_out_assignment() {
   } | \
     mysql_run_query
 
-  # lock every loop this package breaks
-  find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-    -regex '.*/loop_[0-9]\+' \
-    -exec grep -qxF "$1" '{}' \; \
-    -exec touch '{}.locked' \;
-
   exit 0
 
 }
diff --git a/bin/get-package-updates b/bin/get-package-updates
index a6c3f32..69c5e9b 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -528,7 +528,7 @@ echo 'Now actually sort it.'
 
 {
   # this part will have the correct build order, but all the infos are missing
-  tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \
+  tsort "${work_dir}/build-order" | \
     nl -ba | \
     awk '{print $1 " not-git also-not-git whatever " $2}'
   # this part has all the infos, but possibly the wrong order
@@ -543,43 +543,6 @@ echo 'Now actually sort it.'
   awk '{print $5 " " $7 " " $8 " " $9}' > \
   "${work_dir}/build-list.new.new"
 
-rm --one-file-system -rf "${work_dir}/build-list.loops.new"
-mkdir "${work_dir}/build-list.loops.new"
-
-if [ -s "${work_dir}/tsort.error" ]; then
-  >&2 echo 'WARNING: There is a dependency cycle!'
-  >&2 cat "${work_dir}/tsort.error"
-  >&2 echo
-  >&2 echo 'I will continue anyway.'
-  # save loops in separate files each, so breaking them is easier
-  awk '
-    /^tsort: \S+: input contains a loop:$/{
-      n++;
-      getline
-    }
-    {
-      print $2 >"'"${work_dir}"'/build-list.loops.new/loop_" n
-    }
-    ' "${work_dir}/tsort.error"
-
-  # remove lines from loop files which are no packages
-  find "${work_dir}/build-list.loops.new" -maxdepth 1 -regextype grep \
-    -regex '.*/loop_[0-9]\+' \
-    -printf '%p\n' | \
-    while read -r loop; do
-      {
-        sort -u "${loop}"
-        cut -d' ' -f1 "${work_dir}/build-list.new.new" | \
-          sort -u
-      } | \
-        sort | \
-        uniq -d | \
-        sponge "${loop}"
-    done
-else
-  rm "${work_dir}/tsort.error"
-fi
-
 # shellcheck disable=SC2016
 {
   # update hashes of repositories in mysql database
@@ -608,9 +571,8 @@ mysql_find_build_assignment_loops
 
 # Move the .new-files to the actual files
 
-rm -rf --one-file-system "${work_dir}/build-list.loops"
 {
-  printf '%s\n' "build-list.loops" "build-list.new" "build-list" "deletion-list"
+  printf '%s\n' "build-list.new" "build-list" "deletion-list"
   # shellcheck disable=SC2086
   printf '%s.revision\n' ${repo_names}
 } | \
diff --git a/bin/return-assignment b/bin/return-assignment
index c259f1c..b66244b 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -169,18 +169,6 @@ if [ "$5" = 'ERROR' ]; then
   if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
     rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
 
-    # unlock every loop this package would have broken and which is not
-    # broken by another locked package
-    locked_packages=$(
-      find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' -printf '%f\n' | \
-        sed 's@^\(.\+\)\.\([0-9a-f]\{40\}\.\)\{2\}[^.]\+\.locked$@\1@'
-    )
-    find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-      -regex '.*/loop_[0-9]\+' \
-      -exec grep -qxF "$1" '{}' \; \
-      -not -exec grep -qxF "${locked_packages}" '{}' \; \
-      -exec rm '{}.locked' \;
-
     # move that build order to the end of the build-list
     sed -i '
       /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
@@ -538,12 +526,6 @@ find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
   -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \
   -exec rm '{}' \;
 
-# remove all loops which are broken by this package
-find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-  -regex '.*/loop_[0-9]\+' \
-  -exec grep -qxF "$1" '{}' \; \
-  -exec rm '{}.locked' \;
-
 # remove package from build list
 sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
 
diff --git a/conf/default.conf b/conf/default.conf
index 7e7ed3d..b508deb 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -135,7 +135,6 @@ fi
 mkdir -p "${work_dir}"
 touch "${work_dir}/build-list"
 touch "${work_dir}/deletion-list"
-mkdir -p "${work_dir}/build-list.loops"
 
 for repo in ${repo_names}; do
 
-- 
cgit v1.2.3-54-g00ecf


From f22677d62a722fe23e1ca5a75e78cb645efc2bdb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 14:59:16 +0100
Subject: lib/common-functions: remove unused functions

---
 lib/common-functions | 109 ---------------------------------------------------
 1 file changed, 109 deletions(-)

diff --git a/lib/common-functions b/lib/common-functions
index aa32409..9f78550 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -942,35 +942,6 @@ extract_source_directory() {
 
 }
 
-# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository
-# return a list of packages on the build list which are (run- / build- / check-time)
-# dependencies of the given package
-
-find_dependencies_on_build_list() {
-
-  local package="$1"
-  local git_revision="$2"
-  local mod_git_revision="$3"
-  local repository="$4"
-
-  generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
-  {
-    cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
-    awk '{print $1 "." $2 "." $3 "." $4}' < \
-      "${work_dir}/build-list" | \
-      sed '
-        s|^|'"${work_dir}"'/package-infos/|
-        s|$|\.builds|
-      ' | \
-      xargs -r cat | \
-      sort -u
-  } | \
-    sort | \
-    uniq -d
-
-}
-
 # download_sources_by_hash $package $repository $git_revision $git_mod_revision
 # try to download all sources by their hash into the current directory
 # returns 0 if any source was downloaded and 1 otherwise
@@ -1269,86 +1240,6 @@ smoothen_namcap_log() {
     sponge "${file}"
 }
 
-# print_list_of_archaic_packages $source1 $source2 ...
-# print a list of packages which have not been touched for a while,
-# but which are still in the pipeline, e.g. in $source1, $source2 or ...
-
-print_list_of_archaic_packages() {
-  for source in "$@"; do
-    case "${source}" in
-      'testing')
-        # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \
-          -exec head -n1 {} \; | \
-          "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin
-        # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated
-        # and no longer block other packages from being moved
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \
-          sed '
-            s|\.testing$||
-          '
-      ;;
-      'build-list')
-        while read -r pkg rev mod_rev repo; do
-          git_repo=$(
-            find_repository_with_commit "${rev}"
-          )
-          eval repo_path='"${repo_paths__'"${git_repo}"'}"'
-          commit_date=$(
-            git -C "${repo_path}" show -s --format=%ct "${rev}"
-          )
-          mod_commit_date=$(
-            git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}"
-          )
-          if [ "${mod_commit_date}" -gt "${commit_date}" ]; then
-            commit_date="${mod_commit_date}"
-          fi
-          # packages remaining longer than $max_package_age_build_list days on the build list
-          if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then
-            printf '%s %s %s %s\n' \
-              "${pkg}" \
-              "${rev}" \
-              "${mod_rev}" \
-              "${repo}"
-          fi
-        done < \
-          "${work_dir}/build-list"
-      ;;
-      'staging')
-        # packages remaining longer than $max_package_age_staging days in staging
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \
-          sed '
-            s|\.done$||
-          '
-      ;;
-      *)
-        >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}"
-      ;;
-    esac
-  done | \
-    sort -u
-}
-
-# modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
-# print the given modification revision possibly with a html link to github
-
-modification_revision_link() {
-  local mod_rev="$1"
-  local repo="$2"
-  local pkg="$3"
-
-  if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then
-    printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \
-      "${mod_rev}" \
-      "${repo}" \
-      "${pkg}" \
-      "${mod_rev}"
-  else
-    printf '%s\n' \
-      "${mod_rev}"
-  fi
-}
-
 # trigger_mirror_refreshs
 # trigger a refresh of capable tier 1 mirrors (as backup for master mirror)
 
-- 
cgit v1.2.3-54-g00ecf


From 4465e861c9925f9ed45b4b4229dd712de9915402 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Mar 2018 15:38:16 +0100
Subject: lib/common-functions: decomission even more unused functions

---
 lib/common-functions | 127 ---------------------------------------------------
 1 file changed, 127 deletions(-)

diff --git a/lib/common-functions b/lib/common-functions
index 9f78550..8fc300f 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -1041,133 +1041,6 @@ shrink_version() {
   '
 }
 
-# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ]
-
-# Return (to stdout) the biggest subset A of the packages in $omega whose
-# runtime dependencies in $omega \cup $keep are also in A
-
-# $all_builds and $all_depends either point to an empty file - then they will get
-# filled with cached data for subsequent calls - or to the same files of a previous
-# call
-
-# If non-empty, $force contains packages which are assumed to match the above
-# condition without checking.
-
-# The arguments are names of files with one $package.$revision.$mod_revision.$repository
-# per line.
-
-find_biggest_subset_of_packages() {
-
-  ( # the new shell is intentional
-    omega="$1"
-    keep="$2"
-    all_builds="$3"
-    all_depends="$4"
-    if [ $# -eq 4 ]; then
-      force='/dev/null'
-    elif [ $# -eq 5 ]; then
-      force="$5"
-    else
-      >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#"
-      return 2
-    fi
-
-    if [ ! -s "${all_builds}" ]; then
-      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \
-        -exec sed '
-          s|^|{} |
-          s|^\S\+/||
-          s|\.builds | |
-        ' {} \; | \
-        sort -k2,2 > \
-        "${all_builds}"
-    fi
-
-    if [ ! -s "${all_depends}" ]; then
-      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \
-        -exec sed '
-          s|^|{} |
-          s|^\S\+/||
-          s|\.run-depends | |
-        ' {} \; | \
-        grep -v ' base$' | \
-        sort -k2,2 > \
-        "${all_depends}"
-    fi
-
-    sort -u "${omega}" | \
-      sponge "${omega}"
-
-    temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
-    {
-      sort -u "${keep}"
-      cat "${force}" "${force}"
-    } | \
-      sort | \
-      uniq -u > \
-      "${temp_dir}/keep.new"
-    touch "${temp_dir}/keep"
-
-    while [ -s "${temp_dir}/keep.new" ]; do
-      cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \
-        sort -u | \
-        sponge "${temp_dir}/keep"
-
-      {
-        # we append all packages which are run-dependencies of keep-packages
-        # to the keep-list
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.run-depends|
-        ' "${temp_dir}/keep" | \
-          xargs -r grep -HF '' | \
-          sed '
-            s|^.*/||
-            s|\.run-depends:| |
-          ' | \
-          sort -u | \
-          sort -k2,2 | \
-          uniq -f1 | \
-          join -1 2 -2 2 -o 2.1 - "${all_builds}"
-
-        # we append all packages with run-dependencies on the keep-list
-        # to the keep-list
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.builds|
-        ' "${temp_dir}/keep" | \
-          xargs -r grep -HF '' | \
-          sed '
-            s|^.*/||
-            s|\.builds:| |
-          ' | \
-          sort -u | \
-          sort -k2,2 | \
-          uniq -f1 | \
-          join -1 2 -2 2 -o 2.1 - "${all_depends}"
-      } | \
-        sort -u | \
-        join -1 1 -2 1 -o 2.1 - "${omega}" | \
-        sort -u > \
-        "${temp_dir}/keep.new"
-
-      # "new" is only what has not been there before and what is not forced
-      cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \
-        sort | \
-        uniq -u | \
-        sponge "${temp_dir}/keep.new"
-    done
-
-    cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \
-      sort | \
-      uniq -u
-
-  )
-
-}
-
 # sort_square_bracket_content $file
 # sort the content of [] in $file, print to stdout
 
-- 
cgit v1.2.3-54-g00ecf


From 919d396328617ad13ffe3665d64d83325e078d16 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 09:24:31 +0100
Subject: web-scripts/build-list.php: display which build slaves works on which
 task

---
 web-scripts/build-list.php | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/web-scripts/build-list.php b/web-scripts/build-list.php
index b94970b..5cc7a47 100644
--- a/web-scripts/build-list.php
+++ b/web-scripts/build-list.php
@@ -53,7 +53,8 @@ $result = $mysql -> query(
   "(SELECT count(*) " .
     "FROM `build_dependency_loops` " .
     "WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`" .
-  ") AS `loops` " .
+  ") AS `loops`, " .
+  "`build_slaves`.`name` AS `build_slave` " .
   "FROM `build_assignments` " .
   "JOIN `architectures` ON `build_assignments`.`architecture` = `architectures`.`id` " .
   "JOIN `package_sources` ON `build_assignments`.`package_source` = `package_sources`.`id` " .
@@ -61,6 +62,7 @@ $result = $mysql -> query(
   "JOIN `git_repositories` ON `upstream_repositories`.`git_repository`=`git_repositories`.`id` " .
   "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
   "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
+  "LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id` " .
   "WHERE `repositories`.`name`=\"build-list\"" . $match
 );
 if ($result -> num_rows > 0) {
@@ -162,6 +164,10 @@ if ($result -> num_rows > 0) {
         $row["is_blocked"]
       );
     }
+    if (isset($row["build_slave"]))
+      $rows[$count]["build_slave"] = $row["build_slave"];
+    else
+      $rows[$count]["build_slave"] = "&nbsp;";
     $count++;
   }
 
@@ -186,6 +192,7 @@ if ($result -> num_rows > 0) {
   print "<th>loops</th>";
   print "<th>build error</th>";
   print "<th>blocked</th>";
+  print "<th>handed out to</th>";
   print "</tr>\n";
 
   foreach($rows as $row) {
@@ -200,6 +207,7 @@ if ($result -> num_rows > 0) {
     print "<td>".$row["loops"]."</td>";
     print "<td>".$row["fail_reasons"]."</td>";
     print "<td>".$row["is_blocked"]."</td>";
+    print "<td>".$row["build_slave"]."</td>";
 
     print "</tr>\n";
   }
-- 
cgit v1.2.3-54-g00ecf


From c4bc822a79c1922d28101c86d00ea93fce7a36a5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 09:29:37 +0100
Subject: README.md: the build master needs graphviz

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 985bc5c..334b4bb 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,7 @@ This includes scripts to be run on the build master as well as scripts to be run
 * `base-devel`
 * `bc`
 * `git`
+* `graphviz`
 * `ii`
 * some mysql-server
 * `nginx` or equivalent
-- 
cgit v1.2.3-54-g00ecf


From ddb5d33d2e72984d50ec2df475fc4faee949cad8 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 09:55:58 +0100
Subject: bin/interpret-mail: escape linefeed

---
 bin/interpret-mail | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/interpret-mail b/bin/interpret-mail
index bdcf016..201b221 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -161,7 +161,7 @@ sed -n '
   /\n$/!ba
   s/\n$//
   p
-' "${tmp_dir}/plain-content" |
+' "${tmp_dir}/plain-content" | \
   sed '
     :start_loop
       $!{
-- 
cgit v1.2.3-54-g00ecf


From 3c1f6750e8f906d8230266f904d69ce76d8909b9 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 10:15:31 +0100
Subject: bin/copy-to-build-support: --wait new

---
 bin/copy-to-build-support | 56 ++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 53 insertions(+), 3 deletions(-)

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index b3decd4..404a2b9 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -5,12 +5,62 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# shellcheck disable=SC2016
+usage() {
+  >&2 echo ''
+  >&2 echo 'copy-to-build-support [options] package-list:'
+  >&2 echo '  copy the packages listed in package-list into [build-support]'
+  >&2 echo ''
+  >&2 echo 'possible options:'
+  >&2 echo '  -h|--help:'
+  >&2 echo '    Show this help and exit.'
+  >&2 echo '  -w|--wait:'
+  >&2 echo '    Wait for lock if necessary.'
+  [ -z "$1" ] && exit 1 || exit "$1"
+}
+
+eval set -- "$(
+  getopt -o hw \
+    --long help \
+    --long wait \
+    -n "$(basename "$0")" -- "$@" || \
+    echo usage
+  )"
+
+wait_for_lock='-n'
+
+while true
+do
+  case "$1" in
+    -h|--help)
+      usage 0
+    ;;
+    -w|--wait)
+      wait_for_lock=''
+    ;;
+    --)
+      shift
+      break
+    ;;
+    *)
+      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
+      exit 42
+    ;;
+  esac
+  shift
+done
+
 if [ "$#" -ne 1 ]; then
-  >&2 echo 'usage:'
-  >&2 echo '  copy-to-build-support package-list'
-  exit 2
+  >&2 echo 'No package-list was given.'
+  usage
 fi
 
+exec 9> "${sanity_check_lock_file}"
+flock -s ${wait_for_lock} 9
+
+exec 8> "${package_database_lock_file}"
+flock ${wait_for_lock} 8
+
 tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX")
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-- 
cgit v1.2.3-54-g00ecf


From ec84ed09ff3683eb002765f06d585fb08d9a4671 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 10:17:00 +0100
Subject: bin/modify-package-state: --wait new

---
 bin/modify-package-state | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/bin/modify-package-state b/bin/modify-package-state
index 36fee85..76b8c1e 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -18,25 +18,28 @@ usage() {
   >&2 echo '  -n|--no-report:   Do not report what packages were modified.'
   >&2 echo '  -t|--tested:      Mark package(s) as tested.'
   >&2 echo '  -u|--unblock:     Unblock package(s).'
+  >&2 echo '  -w|--wait:        Wait for lock if necessary.'
   >&2 echo ''
   >&2 echo 'Exactly one of -b|-t|-u is needed for actual operation.'
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
 eval set -- "$(
-  getopt -o bfhntu \
+  getopt -o bfhntuw \
     --long block \
     --long faulty \
     --long help \
     --long no-report \
     --long tested \
     --long unblock \
+    --long wait \
     -n "$(basename "$0")" -- "$@" || \
   echo usage
 )"
 
 action=''
 report=true
+wait_for_lock='-n'
 
 while true
 do
@@ -75,6 +78,9 @@ do
       fi
       action='unblock'
     ;;
+    -w|--wait)
+      wait_for_lock=''
+    ;;
     --)
       shift
       break
@@ -123,13 +129,13 @@ if ! [ -r "${input_file}" ]; then
 fi
 
 exec 9> "${sanity_check_lock_file}"
-if ! flock -s -n 9; then
+if ! flock -s ${wait_for_lock} 9; then
   >&2 echo 'Cannot get sanity-check lock.'
   exit
 fi
 
 exec 8> "${package_database_lock_file}"
-if ! flock -n 8; then
+if ! flock ${wait_for_lock} 8; then
   >&2 echo 'Cannot get package-database lock.'
   exit
 fi
-- 
cgit v1.2.3-54-g00ecf


From bb66461d2a5a6639fa907c93a14a81f3837c3c89 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 10:18:14 +0100
Subject: bin/seed-build-list: --wait new - waiting for lock now optional

---
 bin/seed-build-list | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 341ccc6..ef1ab62 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -27,6 +27,8 @@ usage() {
   >&2 echo '    Do not actually update build-list, just print it.'
   >&2 echo '  -p|--package $pkg_regex:'
   >&2 echo '    Update packages matching $pkg_regex.'
+  >&2 echo '  -w|--wait:'
+  >&2 echo '    Wait for lock if necessary.'
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
@@ -35,13 +37,14 @@ tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
 trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT
 
 eval set -- "$(
-  getopt -o ahi:m:np: \
+  getopt -o ahi:m:np:w \
     --long auto \
     --long help \
     --long ignore: \
     --long mirror: \
     --long no-action \
     --long package: \
+    --long wait \
     -n "$(basename "$0")" -- "$@" || \
     echo usage
   )"
@@ -52,6 +55,7 @@ touch "${tmp_dir}/ignore-packages"
 
 auto=false
 update=true
+wait_for_lock='-n'
 
 while true
 do
@@ -80,6 +84,9 @@ do
       echo "$1" >> \
         "${tmp_dir}/package-regexes"
     ;;
+    -w|--wait)
+      wait_for_lock=''
+    ;;
     --)
       shift
       break
@@ -103,6 +110,15 @@ if [ ! -s "${tmp_dir}/mirrors" ] && \
   exit 0
 fi
 
+# get locks
+if ${update}; then
+  exec 9> "${sanity_check_lock_file}"
+  flock -s ${wait_for_lock} 9
+
+  exec 8> "${build_list_lock_file}"
+  flock ${wait_for_lock} 8
+fi
+
 repos="${stable_package_repositories}"
 
 # genereate must-haves query from mirror delta
@@ -347,13 +363,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
   sort -u | \
   tr '\t' ' ' | \
   if ${update}; then
-    # always block if locked
-    exec 9> "${build_list_lock_file}"
-    flock 9
-
-    exec 8> "${sanity_check_lock_file}"
-    flock -s 8
-
     while read -r pkgbase git_rev mod_git_rev repo; do
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
       generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
-- 
cgit v1.2.3-54-g00ecf


From 9d9efcad1de19425ea79988c9b8dd779999cd73c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 10:19:24 +0100
Subject: bin/interpret-mail: wait for locks on actions

---
 bin/interpret-mail | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bin/interpret-mail b/bin/interpret-mail
index 201b221..83c8999 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -190,7 +190,7 @@ sed -n "$(
 )" "${tmp_dir}/raw-content"
 
 if [ -s "${tmp_dir}/block" ]; then
-  if run_and_log_on_error "${base_dir}/bin/modify-package-state" --block "${tmp_dir}/block"; then
+  if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --block "${tmp_dir}/block"; then
     log 'Successfully blocked %s packages.\n' "$(wc -l < "${tmp_dir}/block")"
   else
     log 'There was an error while blocking the packages - ignoring this message.\n'
@@ -201,7 +201,7 @@ if [ -s "${tmp_dir}/stabilize" ]; then
   sed -i '
     /\.pkg\.tar\.xz$/!s/$/.pkg.tar.xz/
   ' "${tmp_dir}/stabilize"
-  if run_and_log_on_error "${base_dir}/bin/modify-package-state" --tested "${tmp_dir}/stabilize"; then
+  if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --tested "${tmp_dir}/stabilize"; then
     log 'Successfully marked %s packages as tested.\n' "$(wc -l < "${tmp_dir}/stabilize")"
   else
     log 'There was an error while marking the packages as tested - ignoring this message.\n'
@@ -209,7 +209,7 @@ if [ -s "${tmp_dir}/stabilize" ]; then
 fi
 
 if [ -s "${tmp_dir}/unblock" ]; then
-  if run_and_log_on_error "${base_dir}/bin/modify-package-state" --unblock "${tmp_dir}/unblock"; then
+  if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --unblock "${tmp_dir}/unblock"; then
     log 'Successfully unblocked %s packages.\n' "$(wc -l < "${tmp_dir}/unblock")"
   else
     log 'There was an error while unblocking the packages - ignoring this message.\n'
@@ -218,7 +218,7 @@ fi
 
 if [ -s "${tmp_dir}/schedule" ]; then
   # shellcheck disable=SC2046
-  "${base_dir}/bin/seed-build-list" $(
+  "${base_dir}/bin/seed-build-list" --wait $(
     tr '[:space:]' '\n' < \
       "${tmp_dir}/schedule" | \
       grep -vxF '' | \
@@ -234,7 +234,7 @@ if [ -s "${tmp_dir}/copy-to-build-support" ]; then
   sed -i '
     /\.pkg\.tar\.xz$/!s/$/.pkg.tar.xz/
   ' "${tmp_dir}/copy-to-build-support"
-  if run_and_log_on_error "${base_dir}/bin/copy-to-build-support" "${tmp_dir}/copy-to-build-support"; then
+  if run_and_log_on_error "${base_dir}/bin/copy-to-build-support" --wait "${tmp_dir}/copy-to-build-support"; then
     log 'Successfully copied %s packages to [build-support].\n' "$(wc -l < "${tmp_dir}/copy-to-build-support")"
   else
     log 'There was an error while copying the packages to [build-support] - ignoring this message.\n'
-- 
cgit v1.2.3-54-g00ecf


From 944da1b00e72f35fd758951a7b25bbb61f43b9fb Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 10:23:52 +0100
Subject: bin/get-package-updates: work/build-order decomissioned

---
 bin/get-package-updates | 39 ++-------------------------------------
 1 file changed, 2 insertions(+), 37 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 69c5e9b..ce9d57a 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -506,42 +506,7 @@ if [ -n "${test_exclusion}" ]; then
   exit 0
 fi
 
-# Now we create the partial order.
-
-while read -r package git_revision mod_git_revision repository; do
-  # add "$pkgname -> $build-target" to build-order list
-  sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.builds"
-  # add "$dependency -> $pkgname" to build-order list
-  sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
-  # add "base/base-devel -> $pkgname" to build-order list
-  printf '%s '"${package}"'\n' 'base' 'base-devel'
-done \
-  < "${work_dir}/build-list.new" \
-  > "${work_dir}/build-order"
-
-if grep -vq '^\S\+ \S\+$' "${work_dir}/build-order"; then
-  >&2 echo 'Created partial order file "build-order" is invalid.'
-  exit 2
-fi
-
-echo 'Now actually sort it.'
-
-{
-  # this part will have the correct build order, but all the infos are missing
-  tsort "${work_dir}/build-order" | \
-    nl -ba | \
-    awk '{print $1 " not-git also-not-git whatever " $2}'
-  # this part has all the infos, but possibly the wrong order
-  awk '{print "0 " $2 " " $3 " " $4 " " $1}' "${work_dir}/build-list.new"
-} | \
-  sort -k5,5 -k1nr | \
-  # now, we have the correct order and the infos, but in adjacent lines
-  uniq -f4 -D | \
-  sed '/^0 /d;N;s|\n| |' | \
-  # now in one line, each
-  sort -k1n,1 | \
-  awk '{print $5 " " $7 " " $8 " " $9}' > \
-  "${work_dir}/build-list.new.new"
+echo 'Done - mark decisions as final.'
 
 # shellcheck disable=SC2016
 {
@@ -572,7 +537,7 @@ mysql_find_build_assignment_loops
 # Move the .new-files to the actual files
 
 {
-  printf '%s\n' "build-list.new" "build-list" "deletion-list"
+  printf '%s\n' "build-list" "deletion-list"
   # shellcheck disable=SC2086
   printf '%s.revision\n' ${repo_names}
 } | \
-- 
cgit v1.2.3-54-g00ecf


From 28593a5108321b6c14262f96c322a3be12fee9ba Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 11:16:22 +0100
Subject: bin/ping-from-slave: tell slave if build is still expected from him

---
 bin/ping-from-slave | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/bin/ping-from-slave b/bin/ping-from-slave
index 4013269..afaf018 100755
--- a/bin/ping-from-slave
+++ b/bin/ping-from-slave
@@ -8,4 +8,18 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: acutally do something here :-)
+# TODO: receive/save some statistics about current build
+
+if [ "$(
+  # shellcheck disable=SC2016,SC2154
+  {
+    printf 'SELECT count(*) FROM `build_slaves`'
+    mysql_join_build_slaves_build_assignments
+    printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
+      "$(printf '%s' "${slave}" | base64 -w0)"
+  } | \
+    mysql_run_query
+  )" -ne 1 ]; then
+  >&2 echo 'You do not build anything currently - abort whatever you are doing.'
+  exit 2
+fi
-- 
cgit v1.2.3-54-g00ecf


From 47eb9a4ee2a93630dd9d8712b4ea7b3386029cf6 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 12:25:20 +0100
Subject: bin/ping-from-slave new to send pings to buildmaster in regular
 intervals during a build

---
 bin/build-packages | 16 ++++++++++++++++
 bin/ping-to-master | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100755 bin/ping-to-master

diff --git a/bin/build-packages b/bin/build-packages
index c7b5f85..aa3de6d 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -258,6 +258,7 @@ while [ "${count}" -ne 0 ]; do
         fi
         cd "${base_dir}"
         recursively_umount_and_rm "${tmp_dir}"
+        flock -u 9 || true
         exit "${err}"
       }
       find "${work_dir}" \
@@ -371,6 +372,17 @@ while [ "${count}" -ne 0 ]; do
         find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \
           rm {} \;
 
+        echo 'building' > "${tmp_dir}/.ping-build-master"
+        if [ -z "${forced_package}" ]; then
+          # we get a lock on "${work_dir}/ping-build-master.lock",
+          # if we release that lock, ping-to-master should stop _immediately_
+          exec 9> "${work_dir}/ping-build-master.lock"
+          if ! flock -n 9; then
+            >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.'
+            exit 2
+          fi
+          "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" &
+        fi
         >&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) ...' \
           "$(date +'%Y-%m-%d %T')" \
           "${package}" \
@@ -390,6 +402,7 @@ while [ "${count}" -ne 0 ]; do
           >&2 printf ' ok.\n'
           tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX")
           find . -maxdepth 1 -type f -name '*-debug-*.pkg.tar.xz*' -delete
+          echo 'post-build' > "${tmp_dir}/.ping-build-master"
           >&2 printf 'signing package(s)\n'
           find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
             -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \
@@ -501,6 +514,7 @@ while [ "${count}" -ne 0 ]; do
               fi
             done
           >&2 printf ' ok.\n'
+          echo 'uploading' > "${tmp_dir}/.ping-build-master"
           if ${upload_to_build_master}; then
             find "${tar_content_dir}/" -maxdepth 1 \
               \( \
@@ -565,6 +579,7 @@ while [ "${count}" -ne 0 ]; do
           success=true
           break
         fi
+        echo 'failure' > "${tmp_dir}/.ping-build-master"
         >&2 printf ' failed.\n'
       done
 
@@ -619,6 +634,7 @@ while [ "${count}" -ne 0 ]; do
       # clean up tmp_dir
       cd "${base_dir}"
       recursively_umount_and_rm "${tmp_dir}"
+      flock -u 9 || true
       trap - EXIT
 
       continue
diff --git a/bin/ping-to-master b/bin/ping-to-master
new file mode 100755
index 0000000..15c9120
--- /dev/null
+++ b/bin/ping-to-master
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# periodically connects to the buildmaster to call ping-from slave to:
+#  - report any update on the build process
+#  - show that the build is still running
+#  - get notified by the build master if the build is not necessary anymore
+
+# shellcheck source=conf/default.conf
+. "${0%/*}/../conf/default.conf"
+
+# TODO: abort build if requested to
+
+parent_pid="$1"
+parent_tmp_dir="$2"
+
+exec 9> "${work_dir}/ping-build-master.lock"
+
+while kill -0 "${parent_pid}" && \
+  [ -f "${parent_tmp_dir}/.ping-build-master" ]; do
+
+  # shellcheck disable=SC2029
+  find "${parent_tmp_dir}" \
+    -xdev \
+    -type f \
+    -name '*.build-log' \
+    -exec wc -l {} \; | \
+    sed 's, .*/, ,' | \
+    ssh \
+      -i "${master_build_server_identity}" \
+      -p "${master_build_server_port}" \
+      "${master_build_server_user}@${master_build_server}" \
+      'ping-from-slave' "$(cat "${parent_tmp_dir}/.ping-build-master")"
+
+  # we wait upto 60 seconds to get the lock - if we get it, the parent
+  # must have released it and we're finished
+  if flock -w 60 9; then
+    break
+  fi
+done
-- 
cgit v1.2.3-54-g00ecf


From d85bf410d3c0a441eabd3e7ef7e098eba4c4005a Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 21 Mar 2018 14:30:35 +0100
Subject: bin/build-master-status-from-mysql: clean mysql database before check

---
 bin/build-master-status-from-mysql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index edabc1e..581fd43 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -17,6 +17,9 @@ if ! flock -n 9; then
   exit
 fi
 
+# shellcheck disable=SC2119
+mysql_cleanup
+
 # shellcheck disable=SC2016
 {
   printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`'
-- 
cgit v1.2.3-54-g00ecf


From 70759d4b9e38d8935d8dca40b2bf9e41651a9a07 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 10:30:33 +0100
Subject: bin/repo-copy: make location of tmp_dir configurable via $TMPDIR

---
 bin/repo-copy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/repo-copy b/bin/repo-copy
index 8566d7c..c3450fd 100755
--- a/bin/repo-copy
+++ b/bin/repo-copy
@@ -26,7 +26,7 @@ to_repo="$2"
 shift
 shift
 
-tmp_dir=$(mktemp -d)
+tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/repo-tools.XXXXXXXXXX")
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
 # extract the databases
-- 
cgit v1.2.3-54-g00ecf


From 72c62099827c6fb033cd3b35dd914f1a5e3d0f2b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 10:52:23 +0100
Subject: bin/db-update: extract package database in tmp_dir, not in /tmp's
 tmpfs

---
 bin/db-update | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/db-update b/bin/db-update
index 129a89d..5b0d56f 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -123,6 +123,8 @@ fi
 tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
+export TMPDIR="${tmp_dir}"
+
 for source_stability in 'testing' 'staging'; do
   find "${tmp_dir}" -mindepth 1 -delete
 
-- 
cgit v1.2.3-54-g00ecf


From a10b2cb87acf9b654b8f88924411814c07752fda Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 13:25:34 +0100
Subject: bin/seed-build-list: display error on failed lock

---
 bin/seed-build-list | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index ef1ab62..e01349c 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -113,10 +113,16 @@ fi
 # get locks
 if ${update}; then
   exec 9> "${sanity_check_lock_file}"
-  flock -s ${wait_for_lock} 9
+  if ! flock -s ${wait_for_lock} 9; then
+    >&2 echo 'Cannot get sanity-check lock.'
+    exit 1
+  fi
 
   exec 8> "${build_list_lock_file}"
-  flock ${wait_for_lock} 8
+  if ! flock ${wait_for_lock} 8; then
+    >&2 echo 'Cannot get build-list lock.'
+    exit 1
+  fi
 fi
 
 repos="${stable_package_repositories}"
-- 
cgit v1.2.3-54-g00ecf


From 999bf6f69012624033211d94bd731a06f4385f47 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 13:42:18 +0100
Subject: bin/seed-build-list: print name to stdout, too, so return-assignment
 reports correcly about rescheduled packages

---
 bin/seed-build-list | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/seed-build-list b/bin/seed-build-list
index e01349c..0db9cdf 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -371,6 +371,8 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
   if ${update}; then
     while read -r pkgbase git_rev mod_git_rev repo; do
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
+      printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \
+        sed 's/ $/\n/'
       generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \
-- 
cgit v1.2.3-54-g00ecf


From da313910bc4dd8478855c9320541c71bd258c948 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 13:46:26 +0100
Subject: bin/db-update -p may now also move untested packages (and break
 consistency)

---
 bin/db-update | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/bin/db-update b/bin/db-update
index 5b0d56f..6ecf9c9 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -185,12 +185,6 @@ for source_stability in 'testing' 'staging'; do
       mysql_join_repositories_repository_stabilities 'subst_r' 'subst_rs'
       printf ' AND `subst_rs`.`name`="%s"' \
         "${source_stability}"
-      # TODO: we may relax this condition, if we a) got rid of state files
-      # or b) improved keeping them in sync (*.testing -> /dev/null is not
-      # implemented, only .tested -> /dev/null)
-      if [ ! "${source_stability}" = 'staging' ]; then
-        printf ' AND `subst_bp`.`is_tested`'
-      fi
       mysql_join_binary_packages_dependencies
       mysql_join_dependencies_dependency_types
       printf ' AND `dependency_types`.`relevant_for_binary_packages`'
-- 
cgit v1.2.3-54-g00ecf


From 5d7f3bf6df321253e9e2718fa2cc0552793268b2 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:04:57 +0100
Subject: bin/why-dont-you: decommission actions that do not look into the
 database

---
 bin/why-dont-you | 269 +------------------------------------------------------
 1 file changed, 4 insertions(+), 265 deletions(-)

diff --git a/bin/why-dont-you b/bin/why-dont-you
index 64dfa1b..e873c71 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -7,7 +7,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: read information from database
+# TODO: reintrocude "keep", "stubbornly_keep", "stabilize" and "unstage"
+# using information from the database.
 
 action="$1"
 shift
@@ -170,275 +171,13 @@ case "${action}" in
 
   'stabilize'|'unstage')
 
-    if [ "${action}" = 'stabilize' ]; then
-      suffix='tested'
-    else
-      suffix='done'
-    fi
-
-    {
-      awk '{print $1 "." $2 "." $3 "." $4}' < \
-        "${work_dir}/build-list"
-      if [ "${action}" = 'stabilize' ]; then
-        find "${work_dir}/package-states" -maxdepth 1 \( \
-          -name '*.done' -o \
-          -name '*.testing' \
-        \) -printf '%f\n' | \
-          sed 's|\.[^.]\+$||'
-      fi
-    } | \
-      sort -u > \
-      "${tmp_dir}/unmoveable-list"
-
-    find "${work_dir}/package-states" -maxdepth 1 -name "*.${suffix}" -printf '%f\n' | \
-      sed 's|\.[^.]\+$||' | \
-      sort -u > \
-      "${tmp_dir}/moveable-list"
-
-    cat "${tmp_dir}/moveable-list" "${tmp_dir}/unmoveable-list" | \
-      sed '
-        s|^|'"${work_dir}/package-infos/"'|
-        s|$|.run-depends|
-      ' | \
-      # base is boring, so we ignore it (might give result "... can be unstaged" although it _will_not_ be unstaged!
-      xargs -r grep -vHxF 'base' | \
-      sed '
-        s|^[^:]*/||
-        s|\.run-depends:| |
-      ' | \
-      sort -k2,2 > \
-      "${tmp_dir}/all-run-depends"
-
-    cat "${tmp_dir}/moveable-list" "${tmp_dir}/unmoveable-list" | \
-      sed '
-        s|^|'"${work_dir}/package-infos/"'|
-        s|$|.builds|
-      ' | \
-      # base is boring, so we ignore it (might give result "... can be unstaged" although it _will_not_ be unstaged!
-      xargs -r grep -vHxF 'base' | \
-      sed '
-        s|^[^:]*/||
-        s|\.builds:| |
-      ' | \
-      sort -k2,2 > \
-      "${tmp_dir}/all-builds"
-
-    for pkg in "$@"; do
-
-      if ! state_file=$(
-          grep '^'"$(str_to_regex "${pkg}")"'\(\.[^.]\+\)\{3\}$' "${tmp_dir}/moveable-list"
-        ) || \
-        [ ! -f "${work_dir}/package-states/${state_file}.${suffix}" ]; then
-        printf '"%s" is not %s yet!\n' "${pkg}" "${suffix}"
-        continue
-      fi
-
-      echo "${state_file}" > "${tmp_dir}/dependent.new"
-      touch "${tmp_dir}/dependent"
-
-      while [ -s "${tmp_dir}/dependent.new" ]; do
-        cat "${tmp_dir}/dependent.new" "${tmp_dir}/dependent" | \
-          sort -u | \
-          sponge "${tmp_dir}/dependent"
-
-        {
-          sed '
-            s|^|'"${work_dir}"'/package-infos/|
-            s|$|.builds|
-          ' "${tmp_dir}/dependent.new" | \
-            xargs -r cat | \
-            sort -u | \
-            join -1 1 -2 2 -o 2.1 - "${tmp_dir}/all-run-depends"
-          sed '
-            s|^|'"${work_dir}"'/package-infos/|
-            s|$|.run-depends|
-          ' "${tmp_dir}/dependent.new" | \
-            xargs -r cat | \
-            sort -u | \
-            join -1 1 -2 2 -o 2.1 - "${tmp_dir}/all-builds"
-        } | \
-          sort -u | \
-          sponge "${tmp_dir}/dependent.new"
-
-        cat "${tmp_dir}/dependent.new" "${tmp_dir}/dependent" "${tmp_dir}/dependent" | \
-          sort | \
-          uniq -u | \
-          sponge "${tmp_dir}/dependent.new"
-      done
-
-      dependent_unmoveable=$(
-        join -1 1 -2 1 -o 2.1 "${tmp_dir}/unmoveable-list" "${tmp_dir}/dependent"
-      )
-
-      if [ -n "${dependent_unmoveable}" ]; then
-        printf 'The following packages are dependent on "%s", but cannot be %sd:\n' "${pkg}" "${action}"
-        echo "${dependent_unmoveable}" | \
-          while read -r sf; do
-            printf '%s' "${sf}"
-            if [ -f "${work_dir}/package-states/${sf}.testing" ]; then
-              printf ' (not tested yet for %s days)' "$(( ($(date '+%s') - $(stat -c '%Y' "${work_dir}/package-states/${sf}.testing")) / 3600 / 24 ))"
-            elif [ -f "${work_dir}/package-states/${sf}.done" ]; then
-              printf ' (not unstaged yet for %s days)' "$(( ($(date '+%s') - $(stat -c '%Y' "${work_dir}/package-states/${sf}.done")) / 3600 / 24 ))"
-            elif tr ' ' '.' < \
-              "${work_dir}/build-list" | \
-              grep -qxF "${sf}"; then
-              printf ' (not built yet)'
-            fi
-            printf '\n'
-          done
-        printf '\n'
-        continue
-      fi
-
-      printf 'Package "%s" can be %sd.\n' "${pkg}" "${action}"
-
-    done
+    printf 'Sry, "why-dont-you %s" is unavailable, until someone recodes it to look into the database.\n' "${action}"
 
   ;;
 
   'keep'|'stubbornly_keep')
 
-    find '/var/lib/pacman/sync' -name '*.db' -execdir bsdtar -tf '{}' \; | \
-      sed -n '
-        s|-[^-]\+-[^-]\+/$||
-        T
-        p
-      ' | \
-      sort -u > \
-      "${tmp_dir}/upstream-packages"
-
-    while read -r pkg; do
-
-      if builds_file=$(
-        find "${work_dir}/package-infos" -maxdepth 1 -printf '%f\n' | \
-          grep -m1 '^'"$(str_to_regex "${pkg}")"'\(\.[^.]\+\)\{3\}\.builds$'
-        ); then
-
-        builds_file="${builds_file%.*}"
-        prepo="${builds_file##*.}"
-        builds_file="${builds_file%.*}"
-        mod_rev="${builds_file##*.}"
-        builds_file="${builds_file%.*}"
-        rev="${builds_file##*.}"
-
-      else
-
-        found_PKGBUILD=false
-        mod_rev=$(cat "${work_dir}/archlinux32.revision")
-        for repo in ${repo_names}; do
-          eval 'repo_path="${repo_paths__'"${repo}"'}"'
-          rev=$(cat "${work_dir}/${repo}.revision")
-          if [ "${repo}" = 'archlinux32' ]; then
-            if git -C "${repo_path}" archive "${mod_rev}" | \
-              grep -q '^[^/]\+/'"$(str_to_regex "${pkg}")"'/PKGBUILD$'; then
-              prepo=$(
-                git -C "${repo_path}" archive "${mod_rev}" | \
-                  grep '^[^/]\+/'"$(str_to_regex "${pkg}")"'/PKGBUILD$' | \
-                  cut -d/ -f1
-              )
-              found_PKGBUILD=true
-              break
-            fi
-          else
-            prepo=$(
-              git -C "${repo_path}" archive "${rev}" -- "${pkg}/repos" 2>/dev/null | \
-                tar -t 2> /dev/null | \
-                grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
-                grep -v -- '-i686/PKGBUILD$' | \
-                grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
-                sed '
-                  s|^[^/]\+/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1|
-                ' | \
-                head -n1
-            )
-            if [ -n "${prepo}" ]; then
-              found_PKGBUILD=true
-              break
-            fi
-          fi
-        done
-        if ! ${found_PKGBUILD}; then
-          continue
-        fi
-
-        generate_package_metadata "${pkg}" "${rev}" "${mod_rev}" "${prepo}"
-
-      fi
-
-      sed "s|^|${pkg} builds |" "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${prepo}.builds"
-
-    done < \
-      "${work_dir}/deletion-list" > \
-      "${tmp_dir}/deleted.builds"
-
-    sort -k3,3 "${tmp_dir}/deleted.builds" | \
-      sponge "${tmp_dir}/deleted.builds"
-
-    for pkg in "$@"; do
-
-      if ! grep -qxF "${pkg}" "${work_dir}/deletion-list"; then
-        printf 'Package "%s" is not on the deletion list.\n' "${pkg}"
-        continue
-      fi
-
-      if ! grep -qxF "${pkg}" "${tmp_dir}/upstream-packages"; then
-        printf 'Package "%s" is not available upstream.\n' "${pkg}"
-        if [ "${action}" = 'keep' ]; then
-          continue
-        fi
-      fi
-
-      if git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
-        tar -Ox 'blacklist' | \
-        sed '
-          s/\s*#.*$//
-          /^\s*$/d
-        ' | \
-        grep -qxF "${pkg}"; then
-        printf 'Package "%s" is explicitely blacklisted.\n' "${pkg}"
-        continue
-      fi
-
-      if [ "lib32-${pkg#lib32-}" = "${pkg}" ]; then
-        printf 'Package "%s" is a library from multilib.\n' "${pkg}"
-        continue
-      fi
-
-      build_depends=$(
-        find "${work_dir}/package-infos" -maxdepth 1 -name "${pkg}.*.build-depends" -exec cat {} \;
-      )
-      if [ -z "${build_depends}" ]; then
-        printf 'Package "%s" was deleted in the git repositories.\n' "${pkg}"
-        continue
-      fi
-
-      build_depends=$(
-        echo "${build_depends}" | \
-          sort -u
-      )
-
-      errors=$(
-        {
-          # shellcheck disable=SC2086
-          printf '%s\n' ${build_depends}
-          awk '{print $3}' "${tmp_dir}/deleted.builds" | \
-            sort -u
-        } | \
-          sort | \
-          uniq -d | \
-          join -1 1 -2 3 -o 2.1,2.2,2.3 - "${tmp_dir}/deleted.builds"
-      )
-      if [ -n "${errors}" ]; then
-        printf 'Package "%s" has dependencies on the deletion list:\n' "${pkg}"
-        # shellcheck disable=SC2086,SC2183
-        printf '%s %s %s\n' ${errors}
-        printf '\n'
-        continue
-      fi
-
-      printf 'It seems, package "%s" should not be deleted.\n' "${pkg}"
-
-    done
+    printf 'Sry, "why-dont-you %s" is unavailable, until someone recodes it to look into the database.\n' "${action}"
 
   ;;
 
-- 
cgit v1.2.3-54-g00ecf


From 850aa6b864bf0bc4065a49ccca446605f7d7adb4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:09:37 +0100
Subject: bin/build-master-status-from-mysql: do not check build-dependencies -
 they're about to be decommissioned

---
 bin/build-master-status-from-mysql | 53 --------------------------------------
 1 file changed, 53 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 581fd43..0e66473 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -143,55 +143,6 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
   sort -u > \
   "${tmp_dir}/testing.file"
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT '
-  printf '`package_sources`.`%s`,' \
-    'pkgbase' \
-    'git_revision' \
-    'mod_git_revision'
-  printf '`upstream_repositories`.`name`,`install_targets`.`name`'
-  printf ' FROM `binary_packages`'
-  mysql_join_binary_packages_dependencies
-  mysql_join_dependencies_dependency_types
-  mysql_join_dependencies_install_targets
-  mysql_join_binary_packages_build_assignments
-  mysql_join_build_assignments_package_sources
-  mysql_join_package_sources_upstream_repositories
-  printf ' WHERE `dependency_types`.`relevant_for_building`'
-} | \
-  mysql_run_query | \
-  tr '\t' ' ' | \
-  sort -u > \
-  "${tmp_dir}/build-dependencies.mysql"
-{
-  cat "${tmp_dir}/build-dependencies.mysql"
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT DISTINCT '
-    printf '`package_sources`.`%s`,' \
-      'pkgbase' 'git_revision' 'mod_git_revision'
-    printf '`upstream_repositories`.`name`'
-    printf ' FROM `upstream_repositories`'
-    mysql_join_upstream_repositories_package_sources
-    mysql_join_package_sources_build_assignments
-    mysql_join_build_assignments_binary_packages
-  } | \
-    mysql_run_query | \
-    tr '\t' ' ' | \
-    while read -r pkgbase git_revision mod_git_revision repository; do
-      if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then
-        # TODO: consider versions dependencies, too
-        grep -v '[<=>]' "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" | \
-          sed '
-            s/^/'"${pkgbase} ${git_revision} ${mod_git_revision} ${repository}"' /
-          '
-      fi
-    done
-} | \
-  sort -u > \
-  "${tmp_dir}/build-dependencies.file"
-
 {
   diff -u \
     "${tmp_dir}/locked.file" \
@@ -214,10 +165,6 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     "${tmp_dir}/testing.file" \
     "${tmp_dir}/testing.mysql" || \
     true
-  diff -u \
-    "${tmp_dir}/build-dependencies.file" \
-    "${tmp_dir}/build-dependencies.mysql" || \
-    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
-- 
cgit v1.2.3-54-g00ecf


From 7cc6dee8cfd91e44e7d6436e824937a8a0add008 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:15:36 +0100
Subject: lib/common-functions: silence shellcheck

---
 lib/common-functions | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/common-functions b/lib/common-functions
index 8fc300f..e3aab4c 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -908,6 +908,7 @@ find_package_repository_to_package() {
 extract_source_directory() {
 
   local git_repo="$1"
+  # shellcheck disable=SC2034
   local rev="$2"
   local mod_rev="$3"
   local output="$4"
-- 
cgit v1.2.3-54-g00ecf


From 4f5f0fafc4be959a9a05adac8bf5162a7c5bab46 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:18:41 +0100
Subject: lib/common-functions: remove generate_package_metadata,
 delete_old_metadata - this information is now in the database

---
 bin/cleanup             |   3 -
 bin/get-package-updates |   1 -
 bin/seed-build-list     |   1 -
 lib/common-functions    | 228 +-----------------------------------------------
 4 files changed, 1 insertion(+), 232 deletions(-)

diff --git a/bin/cleanup b/bin/cleanup
index 15b6fc9..02ac98b 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -99,9 +99,6 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \
   -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \
   -delete
 
-# remove old package meta data
-delete_old_metadata
-
 # remove dependency graphs of packages on the deletion list
 sed '
   s|^|'"${webserver_directory}"'/graphs/|
diff --git a/bin/get-package-updates b/bin/get-package-updates
index ce9d57a..a36cb8a 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -353,7 +353,6 @@ done | \
       } | \
         mysql_run_query
       mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
-      generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       continue
     fi
 
diff --git a/bin/seed-build-list b/bin/seed-build-list
index 0db9cdf..41707dd 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -373,7 +373,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \
         sed 's/ $/\n/'
-      generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \
         "${work_dir}/build-list"
diff --git a/lib/common-functions b/lib/common-functions
index e3aab4c..0a5ef1c 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -6,7 +6,7 @@
 
 # TODO: have full information (currently in files) in database
 
-# TODO: remove state files / metadata files
+# TODO: remove state files
 
 if [ -z "${base_dir}" ]; then
   # just to make shellcheck happy
@@ -123,232 +123,6 @@ find_git_repository_to_package_repository() {
 
 }
 
-# generate_package_metadata $package $git_revision $mod_git_revision $repository
-# or
-# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# generate the meta data files of a package (dependencies, built packages, ...)
-
-generate_package_metadata() {
-
-  local package="$1"
-  local git_revision="$2"
-  local mod_git_revision="$3"
-  local repository="$4"
-  local file_prefix
-  local file
-  local PKGBUILD
-
-  if [ $# -eq 1 ]; then
-    # second form
-    repository="${package##*.}"
-    package="${package%.*}"
-    mod_git_revision="${package##*.}"
-    package="${package%.*}"
-    git_revision="${package##*.}"
-    package="${package%.*}"
-  fi
-
-  file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
-
-  if [ -e "${file_prefix}.builds" ] && \
-    [ -e "${file_prefix}.build-depends" ] && \
-    [ -e "${file_prefix}.run-depends" ] && \
-    [ -e "${file_prefix}.groups" ] && \
-    [ -e "${file_prefix}.packages" ]; then
-    return 0
-  fi
-
-  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then
-    printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-  if [ ! -s "${file_prefix}.SRCINFO" ]; then
-    >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-
-  # otherwise this just calls for trouble
-  sed -i '
-    /^[^=]*=\s*$/d
-    s/_i686\(\s*=\)/\1/
-  ' "${file_prefix}.SRCINFO"
-
-  # extract "groups" = groups \cup provides
-  grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.groups"
-
-  # extract "packages" = pkgname
-  grep '^pkgname = ' "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.packages"
-
-  # extract "builds" = provides \cup pkgname \cup groups
-  cat "${file_prefix}.groups" "${file_prefix}.packages" | \
-    sort -u > \
-    "${file_prefix}.builds"
-
-  # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base' 'base-devel'
-      sed -n "$(
-        printf '/^\t%s = /p\n' \
-          'depends' \
-          'makedepends'
-      )" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.build-depends"
-
-  # extract "run-depends" = depends \cup \{ base \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base'
-      sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.run-depends"
-
-  rm "${file_prefix}.SRCINFO"
-
-}
-
-# delete_old_metadata
-# delete old (=unneeded) meta data of packages
-
-delete_old_metadata() {
-
-  local current_metadata
-
-  current_metadata=$(
-    find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.[^.]\+$||
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
-      ' | \
-      sort -u
-  )
-
-  ( # the new shell is intentional
-    # what we have
-    echo "${current_metadata}"
-
-    # package-states should stay
-    find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
-      ' | \
-      sort -u | \
-      sed 'p'
-
-    # build-list items should stay
-    sed 'p' "${work_dir}/build-list"
-
-    tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    echo "${current_metadata}" | \
-      sort -k1,1 > \
-      "${tmp_dir}/current-metadata"
-
-    # the newest of the following should stay:
-    {
-      # deletion-list items
-      cat "${work_dir}/deletion-list"
-      # all packages in the repos
-      for repo in ${repo_names}; do
-        eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \
-          tar -t | \
-          sed '
-            s|/$||
-            /\//d
-          '
-      done
-    } | \
-      sort -u | \
-      join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
-      sort -k4,4 > \
-      "${tmp_dir}/find-newest-revisions"
-
-    uniq -uf3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-
-    uniq -Df3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      uniq --group=append -f3 | \
-      {
-        revs=''
-        mod_revs=''
-        opkg=''
-        orepo=''
-        while read -r rev mod_rev repo pkg; do
-
-          if [ -z "${rev}" ] && \
-            [ -z "${mod_rev}" ] && \
-            [ -z "${repo}" ] && \
-            [ -z "${pkg}" ]; then
-
-            printf '%s %s %s %s\n' \
-              "$(
-                printf '%s\n' ${revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "$(
-                printf '%s\n' ${mod_revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "${orepo}" \
-              "${opkg}"
-
-            revs=''
-            mod_revs=''
-            orepo=''
-            opkg=''
-            continue
-          fi
-          revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${revs} ${rev} | \
-              sort -u
-          )
-          mod_revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${mod_revs} ${mod_rev} | \
-              sort -u
-          )
-          orepo="${repo}"
-          opkg="${pkg}"
-        done
-      } | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-  ) | \
-    sort | \
-    uniq -u | \
-    while read -r pkg rev mod_rev repo; do
-      rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
-    done
-}
-
 # repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
 # print which (stable) repository a package belongs to
 
-- 
cgit v1.2.3-54-g00ecf


From 89841624b9eff687dec43c43efba1b5fd54c789e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:36:52 +0100
Subject: bin/cleanup: take info about "recent" logfiles from database

---
 bin/cleanup | 38 ++++++++++++++------------------------
 1 file changed, 14 insertions(+), 24 deletions(-)

diff --git a/bin/cleanup b/bin/cleanup
index 02ac98b..a0bf790 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -42,31 +42,21 @@ fi
 # remove logs where package is not broken/locked anymore
 
 {
-  find "${build_log_directory}/error" -maxdepth 1 -type f -printf '1 %f %f\n' | \
-    sed 's|\.[^. ]\+\.build-log\.gz$||'
-  find "${work_dir}/package-states" -maxdepth 1 \( \
-    -name '*.broken' -o \
-    -name '*.done' -o \
-    -name '*.locked' -o \
-    -name '*.testing' \
-  \) -printf '0 0 %f\n' | \
-    sed 's|\.[^.]\+$||'
+  find "${build_log_directory}/error" -maxdepth 1 -type f -printf 'file %f\n'
+  # shellcheck disable=SC2016
+  printf 'SELECT "mysql",`failed_builds`.`log_file` FROM `failed_builds`;\n' | \
+    mysql_run_query | \
+    tr '\t' ' '
 } | \
-  sort -k3,3 -k1,2 | \
-  uniq --group=prepend -f2 | \
-  while read -r num file _; do
-    if [ "${num}" = '0' ]; then
-      while read -r line; do
-        if [ -z "${line}" ]; then
-          break
-        fi
-      done
-      continue
-    fi
-    if [ -z "${num}${file}" ]; then
-      continue
-    fi
-    rm -f "${build_log_directory}/error/${file}"
+  sort -k2,2 -k1,1 | \
+  uniq -uf 1 | \
+  sed -n '
+    s/^file //
+    T
+    p
+  ' | \
+  while read -r file; do
+    rm "${build_log_directory}/error/${file}"
   done
 
 # only keep 10 newest logs per failed package
-- 
cgit v1.2.3-54-g00ecf


From c13bcbe42ab5b6f020b098f10cdbf8dac76afae4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:39:11 +0100
Subject: bin/build-master-status-from-mysql: do not look for locked testing
 and tested packages (all the information in package-states)

---
 bin/build-master-status-from-mysql | 102 -------------------------------------
 1 file changed, 102 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 0e66473..081baec 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -37,28 +37,6 @@ mysql_cleanup
 sort "${work_dir}/build-list" > \
   "${tmp_dir}/build-list.file"
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT DISTINCT '
-  printf '`package_sources`.`%s`,' \
-    'pkgbase' 'git_revision' 'mod_git_revision'
-  printf '`upstream_repositories`.`name`'
-  printf ' FROM `build_slaves`'
-  mysql_join_build_slaves_build_assignments
-  mysql_join_build_assignments_package_sources
-  mysql_join_package_sources_upstream_repositories
-} | \
-  mysql_run_query | \
-  tr '\t' ' ' | \
-  sort > \
-  "${tmp_dir}/locked.mysql"
-find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \
-  sed '
-    s/\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$/ \1 \2 \3/
-  ' | \
-  sort > \
-  "${tmp_dir}/locked.file"
-
 # shellcheck disable=SC2016
 {
   printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
@@ -75,79 +53,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin
 sort "${work_dir}/deletion-list" > \
   "${tmp_dir}/deletion-list.file"
 
-# shellcheck disable=SC2016
 {
-  printf 'SELECT '
-  printf '`binary_packages`.`%s`,' \
-    'pkgname' \
-    'epoch' \
-    'pkgver' \
-    'pkgrel' \
-    'sub_pkgrel'
-  printf '`architectures`.`name`'
-  printf ' FROM `binary_packages`'
-  mysql_join_binary_packages_repositories
-  mysql_join_binary_packages_architectures
-  mysql_join_repositories_repository_stabilities
-  printf ' WHERE `binary_packages`.`is_tested`'
-  printf ' AND NOT `binary_packages`.`has_issues`'
-  printf ' AND `repository_stabilities`.`name`="testing"'
-} | \
-  mysql_run_query | \
-  sed '
-    s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
-  ' | \
-  sort -u > \
-  "${tmp_dir}/tested.mysql"
-find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
-  -name '*.tested' \
-  -exec cat '{}' \; | \
-  sed '
-    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/
-    s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
-  ' | \
-  sort -u > \
-  "${tmp_dir}/tested.file"
-
-# shellcheck disable=SC2016
-{
-  printf 'SELECT '
-  printf '`binary_packages`.`%s`,' \
-    'pkgname' \
-    'epoch' \
-    'pkgver' \
-    'pkgrel' \
-    'sub_pkgrel'
-  printf '`architectures`.`name`'
-  printf ' FROM `binary_packages`'
-  mysql_join_binary_packages_repositories
-  mysql_join_binary_packages_architectures
-  mysql_join_repositories_repository_stabilities
-  printf ' WHERE NOT `binary_packages`.`is_tested`'
-  printf ' AND NOT `binary_packages`.`has_issues`'
-  printf ' AND `repository_stabilities`.`name`="testing"'
-} | \
-  mysql_run_query | \
-  sed '
-    s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/
-  ' | \
-  sort -u > \
-  "${tmp_dir}/testing.mysql"
-find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
-  -name '*.testing' \
-  -exec cat '{}' \; | \
-  sed '
-    s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/
-    s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
-  ' | \
-  sort -u > \
-  "${tmp_dir}/testing.file"
-
-{
-  diff -u \
-    "${tmp_dir}/locked.file" \
-    "${tmp_dir}/locked.mysql" || \
-    true
   mysql_sanity_check || true
   diff -u \
     "${tmp_dir}/build-list.file" \
@@ -157,14 +63,6 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
     "${tmp_dir}/deletion-list.file" \
     "${tmp_dir}/deletion-list.mysql" || \
     true
-  diff -u \
-    "${tmp_dir}/tested.file" \
-    "${tmp_dir}/tested.mysql" || \
-    true
-  diff -u \
-    "${tmp_dir}/testing.file" \
-    "${tmp_dir}/testing.mysql" || \
-    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
-- 
cgit v1.2.3-54-g00ecf


From 60e7e5a033f0069624bab1754a794e82ec105818 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:56:14 +0100
Subject: bin/sanity-check: compare master mirror to tracking information in
 mysql database

---
 bin/sanity-check | 79 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 39 insertions(+), 40 deletions(-)

diff --git a/bin/sanity-check b/bin/sanity-check
index 9615ac7..3defce2 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -105,7 +105,7 @@ touch "${tmp_dir}/messages"
 trap 'finish' EXIT
 
 if [ $# -eq 0 ]; then
-  set -- git-repositories build-list mysql repos package-database state-files
+  set -- git-repositories build-list mysql repos package-database track-state
 fi
 
 while [ $# -gt 0 ]; do
@@ -337,51 +337,50 @@ while [ $# -gt 0 ]; do
 
     ;;
 
-    state-files)
+    track-state)
 
-      for status in 'staging' 'testing'; do
-
-        [ ${silence} -gt 0 ] || \
-          printf 'checking state-files of "%s" ...' "${status}" >> \
-          "${tmp_dir}/messages"
+      [ ${silence} -gt 0 ] || \
+        printf 'checking if all packages are tracked correctly ...' >> \
+        "${tmp_dir}/messages"
 
-        errors=$(
+      errors=$(
+        {
+          # shellcheck disable=SC2016
           {
-            if [ "${status}" = 'staging' ]; then
-              find "${work_dir}/package-states" -name '*.done' \
-                -exec sed 's|^|package-state-file |' '{}' \;
-            else
-              find "${work_dir}/package-states" \( -name '*.testing' -o -name '*.tested' \) \
-                -exec sed 's|^|package-state-file |' '{}' \;
-            fi
-            ls_master_mirror 'i686' | \
-              grep "${status}\$" | \
-              while read -r repo; do
-                ls_master_mirror "i686/${repo}"
-              done | \
-              grep '\.pkg\.tar\.xz$' | \
-              sed 's|^|package-file |'
+            printf 'SELECT "mysql",CONCAT(`repositories`.`name`,"/",'
+            mysql_package_name_query
+            printf ') FROM `binary_packages`'
+            mysql_join_binary_packages_repositories
+            printf ' AND `repositories`.`is_on_master_mirror`'
+            mysql_join_binary_packages_architectures
           } | \
-            sort -k2 | \
-            uniq -cf1 | \
-            grep -v '^\s*2\s' | \
-            awk '{print $2 " " $3}'
-        )
-        if [ -n "${errors}" ]; then
-          if [ ${silence} -le 1 ]; then
-            printf '\nThe following %s packages do not have state files or vice versa:\n%s\n' \
-              "${status}" \
-              "${errors}" >> \
-              "${tmp_dir}/messages"
-          fi
-          i_am_insane
+            mysql_run_query | \
+            tr '\t' ' '
+          ls_master_mirror 'i686' | \
+            while read -r repo; do
+              ls_master_mirror "i686/${repo}" | \
+                sed '
+                  /\.pkg\.tar\.xz$/!d
+                  s,^,package-file '"${repo}"'/,
+                '
+            done
+        } | \
+          sed 's/\(-[0-9]\+\)\.0\(-[^- ]\+$\)/\1\2/' | \
+          sort -k2 | \
+          uniq -uf1
+      )
+      if [ -n "${errors}" ]; then
+        if [ ${silence} -le 1 ]; then
+          printf '\nThe following packages from the master mirror are not tracked in the database or vice versa:\n%s\n' \
+            "${errors}" >> \
+            "${tmp_dir}/messages"
         fi
+        i_am_insane
+      fi
 
-        [ ${silence} -gt 0 ] || \
-          echo ' passed.' >> \
-          "${tmp_dir}/messages"
-
-      done
+      [ ${silence} -gt 0 ] || \
+        echo ' passed.' >> \
+        "${tmp_dir}/messages"
 
     ;;
 
-- 
cgit v1.2.3-54-g00ecf


From d6ac133a9b477c2b6d606937033d9bc47b5c1aef Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:00:44 +0100
Subject: bin/return-assignment: make "haskell-" prefix optional for
 rescheduling packages

---
 bin/return-assignment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index b66244b..02bd491 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -196,7 +196,7 @@ if [ "$5" = 'ERROR' ]; then
         ' | \
         tr ' ' '\n' | \
         sed '
-          s/^/-p ^haskell-/
+          s/^/-p ^\\(haskell-\\)\\?/
           s/-[0-9.]\+$/\$/
         ' | \
         sort -u
-- 
cgit v1.2.3-54-g00ecf


From 1719184bafa03d92bc8eae895704aa446e0bd587 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:11:14 +0100
Subject: bin/sanity-check: remove done TODO

---
 bin/sanity-check | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/sanity-check b/bin/sanity-check
index 3defce2..4398595 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -5,8 +5,6 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: read information from database
-
 usage() {
   >&2 echo ''
   >&2 echo 'sanity-check [options] [checks]: check sanity of build master'
-- 
cgit v1.2.3-54-g00ecf


From e808f93c3f9af51fa32764d3195bbba33b73d026 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:12:37 +0100
Subject: bin/build-packages: remove done TODO

---
 bin/build-packages | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bin/build-packages b/bin/build-packages
index aa3de6d..02334e9 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -7,9 +7,6 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: report back in regular intervals, so the build master can
-# abort if the package is outdated and/or someone else finished it.
-
 # TODO: build other 'architectures', too (pentium4, i486)
 
 # shellcheck disable=SC2016
-- 
cgit v1.2.3-54-g00ecf


From c772be57eca3a2176ac3931c02fb582f85c0d575 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:19:23 +0100
Subject: decommission package-states directory - we rely on the database now

---
 bin/bootstrap-mysql      | 60 ++++------------------------------------------
 bin/cleanup              | 20 ----------------
 bin/db-update            | 62 ------------------------------------------------
 bin/get-assignment       | 27 +++------------------
 bin/get-package-updates  |  6 -----
 bin/modify-package-state | 38 -----------------------------
 bin/return-assignment    | 38 -----------------------------
 lib/common-functions     |  4 ----
 8 files changed, 7 insertions(+), 248 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 4c1938f..fc347d7 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -3,6 +3,10 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: this should become some sort of mysql dump to bootstrap the
+# tables and stored functions only - when the database is gone, we're
+# pretty much screwed anyway :-D
+
 tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
@@ -792,68 +796,12 @@ fi
 } | \
   mysql_run_query
 
-find "${work_dir}/package-states" \
-  -maxdepth 1 \
-  -mindepth 1 \
-  \( \
-    -name '*.done' -o \
-    -name '*.testing' -o \
-    -name '*.tested' \
-  \) \
-  -exec sed '
-    1!d
-    s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@
-    s@^\S*/@@
-    s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 /
-    s/ \([0-9]\+\) / \1.0 /
-    s/ [0-9]\+\.\([0-9]\+\) / \1 /
-  ' {} \; | \
-  while read -r state_file sub_pkgrel arch package_file; do
-    printf '%s ' "${state_file}"
-    repository=$(
-      grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \
-        cut -d' ' -f1
-    )
-    if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then
-      printf 'not found exactly once on the master mirror.\n'
-      >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}"
-      continue
-    fi
-    mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}"
-    printf ' ok\n'
-  done
-
 tr ' ' '.' < \
   "${work_dir}/build-list" | \
   while read -r state_file; do
     mysql_generate_package_metadata 'build-list' "${state_file}"
   done
 
-# shellcheck disable=SC2016
-find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \
-  sed '
-    s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3|
-  ' | \
-  while read -r state_file pkgbase git_revision mod_git_revision repository; do
-    printf 'UPDATE `build_assignments`'
-    mysql_join_build_assignments_package_sources
-    mysql_join_package_sources_upstream_repositories
-    printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \
-      "$(
-        tr -d '\n' < \
-          "${state_file}" | \
-          base64 -w0
-      )"
-    printf ' WHERE'
-    printf ' `package_sources`.`%s`=from_base64("%s") AND' \
-      'pkgbase'          "$(printf '%s' "${pkgbase}" |          base64 -w0)" \
-      'git_revision'     "$(printf '%s' "${git_revision}" |     base64 -w0)" \
-      'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)"
-    printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
-      "$(printf '%s' "${repository}" | base64 -w0)"
-  done | \
-  mysql_run_query
-
 grep '^\('"$(
   # shellcheck disable=SC2086
   printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories}
diff --git a/bin/cleanup b/bin/cleanup
index a0bf790..7aaa39f 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -19,26 +19,6 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
-# remove blocked/broken/locked markes of packages not on the buildlist anymore
-
-{
-  find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \
-    sed '
-      s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5|
-    '
-  awk '{
-    print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken"
-    print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked"
-    print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked"
-  }' "${work_dir}/build-list"
-} | \
-  sort -k3 | \
-  uniq -uf2 | \
-  grep '^state ' | \
-  awk '{print $2}' | \
-  sed "s|^|${work_dir}/package-states/|" | \
-  xargs -rn1 rm
-
 # remove logs where package is not broken/locked anymore
 
 {
diff --git a/bin/db-update b/bin/db-update
index 6ecf9c9..6722f24 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -207,12 +207,6 @@ for source_stability in 'testing' 'staging'; do
       mysql_join_repositories_repository_stabilities
       printf ' AND `repository_stabilities`.`name`="%s"' \
         "${source_stability}"
-      # TODO: we may relax this condition, if we a) got rid of state files
-      # or b) improved keeping them in sync (*.testing -> /dev/null is not
-      # implemented, only .tested -> /dev/null)
-      if [ ! "${source_stability}" = 'staging' ]; then
-        printf ' AND `binary_packages`.`is_tested`'
-      fi
       mysql_join_binary_packages_build_assignments
       mysql_join_build_assignments_package_sources
       mysql_join_package_sources_upstream_repositories
@@ -401,62 +395,6 @@ for source_stability in 'testing' 'staging'; do
     printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n'
   } | \
     mysql_run_query
-
-  sed_apply_rm=$(
-    while read -r pkg _ _; do
-      printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
-    done < \
-      "${tmp_dir}/rm"
-  )
-
-  sed_apply_remove_part_of_mv=$(
-    while read -r pkg _ _; do
-      printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
-    done < \
-      "${tmp_dir}/mv"
-  )
-
-  sed_apply_move_part_of_mv=$(
-    while read -r pkg _ _; do
-      printf '/^%s$/{\n' "$(str_to_regex "${pkg}")"
-      printf 'w %%s\n'
-      printf 'd\n'
-      printf '}\n'
-    done < \
-      "${tmp_dir}/mv"
-  )
-
-  find "${work_dir}/package-states/" \
-    \( \
-      -name '*.done' -o \
-      -name '*.testing' -o \
-      -name '*.tested' \
-    \) \
-    -exec sed -i "${sed_apply_rm}" '{}' \;
-  find "${work_dir}/package-states/" \
-    -name '*.tested' \
-    -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \;
-  find "${work_dir}/package-states/" \
-    -name '*.done' \
-    -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
-    -printf '%p\n' | \
-    while read -r sf; do
-      sed -i "$(
-        printf '%s\n' "${sed_apply_move_part_of_mv}" | \
-          sed '
-            s,%s,'"${sf%.done}.testing"',g
-          '
-      )" "${sf}"
-    done
-  find "${work_dir}/package-states/" \
-    \( \
-      -name '*.done' -o \
-      -name '*.testing' -o \
-      -name '*.tested' \
-    \) \
-    -type f \
-    -not -exec test -s '{}' \; \
-    -delete
 done
 
 trigger_mirror_refreshs
diff --git a/bin/get-assignment b/bin/get-assignment
index 3357fc6..a82e1f8 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -18,17 +18,8 @@
 # TODO: honor manual build order of tool-chain:
 # toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc
 
-mkdir -p "${work_dir}/package-states"
-
 hand_out_assignment() {
 
-  # we don't care anymore if an older version of this package was
-  # "locked" or "broken"
-  find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
-    -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \
-    -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \
-    -delete
-
   # move that build order to the end of the build-list
   sed -i '
     /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
@@ -57,21 +48,7 @@ hand_out_assignment() {
     printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
       "$(printf '%s' "$4" | base64 -w0)"
     printf ' LIMIT 1;\n'
-  } | \
-    mysql_run_query | \
-    tr '\t' ' '
 
-  {
-    # shellcheck disable=SC2154
-    echo "${slave}"
-    if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
-      cat "${work_dir}/package-states/$1.$2.$3.$4.locked"
-    fi
-  } | \
-    sort -u | \
-    sponge "${work_dir}/package-states/$1.$2.$3.$4.locked"
-  # shellcheck disable=SC2016
-  {
     printf 'UPDATE `build_slaves`'
     printf ' SET `currently_building` = ('
     printf ' SELECT `build_assignments`.`id`'
@@ -86,6 +63,7 @@ hand_out_assignment() {
     printf ' `upstream_repositories`.`name` = from_base64("%s")' \
       "$(printf '%s' "$4" | base64 -w0)"
     printf ')'
+    # shellcheck disable=SC2154
     printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
       "$(printf '%s' "${slave}" | base64 -w0)"
 
@@ -101,7 +79,8 @@ hand_out_assignment() {
     printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \
       "$(printf '%s' "$4" | base64 -w0)"
   } | \
-    mysql_run_query
+    mysql_run_query | \
+    tr '\t' ' '
 
   exit 0
 
diff --git a/bin/get-package-updates b/bin/get-package-updates
index a36cb8a..3e9edf9 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -111,9 +111,6 @@ delete_package() {
     mysql_run_query >> \
     "${work_dir}/deletion-list.new"
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
-  find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \
-    -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \
-    -delete
   # TODO: Once we want to rely on the database for test_exclusion, we
   # need to run the command below unconditionally, but with some
   # changes, so we can easily revert.
@@ -317,9 +314,6 @@ done | \
     if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
       # new or modified PKGBUILD
       sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
-      find "${work_dir}/package-states" -regextype grep \
-        -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \
-        -delete
       # shellcheck disable=SC2154
       echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
         "${work_dir}/build-list.new"
diff --git a/bin/modify-package-state b/bin/modify-package-state
index 76b8c1e..4a2f5ac 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -243,46 +243,8 @@ while read -r package reason; do
     {
       printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}"
       printf ';\n'
-      # TODO: remove the below part, once the database if fully functional
-      printf 'UPDATE `binary_packages` AS `a`'
-      printf ' JOIN `binary_packages` AS `b`'
-      printf ' ON `a`.`build_assignment`=`b`.`build_assignment`'
-      mysql_join_binary_packages_repositories 'b'
-      printf ' SET `b`.`is_tested`=1'
-      printf ' WHERE `a`.`is_tested`'
-      printf ' AND NOT `b`.`is_tested`'
-      printf ' AND NOT `repositories`.`name`="deletion-list";\n'
     } | \
       mysql_run_query
-    case "${action}" in
-      'block')
-        printf '%s' "${reason}" > \
-          "${work_dir}/package-states/${package}.blocked"
-      ;;
-      'unblock')
-        rm -f "${work_dir}/package-states/${package}.blocked"
-      ;;
-      'faulty')
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
-          -name '*.tested' \
-          -exec grep -lxF "${package}" '{}' \; | \
-          while read -r sf; do
-            mv "${sf}" "${sf%ed}ing"
-          done
-      ;;
-      'tested')
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
-          -name '*.testing' \
-          -exec grep -lxF "${package}" '{}' \; | \
-          while read -r sf; do
-            mv "${sf}" "${sf%ing}ed"
-          done
-      ;;
-      *)
-        >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
-        exit 42
-      ;;
-    esac
     printf '%s %s\n' "${action}" "${package}"
   fi
 done \
diff --git a/bin/return-assignment b/bin/return-assignment
index 02bd491..da9e762 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -89,9 +89,6 @@ if [ "$5" = 'ERROR' ]; then
       '*.build-log.gz'
   )
 
-  # shellcheck disable=SC2154
-  echo "${slave}" >> \
-    "${work_dir}/package-states/$1.$2.$3.$4.broken"
   # shellcheck disable=SC2016
   {
     if [ -n "${saved_build_logs}" ]; then
@@ -162,23 +159,6 @@ if [ "$5" = 'ERROR' ]; then
   } | \
     mysql_run_query
 
-  # shellcheck disable=SC2154
-  sed -i '
-    /^'"$(str_to_regex "${slave}")"'$/d
-  ' "${work_dir}/package-states/$1.$2.$3.$4.locked"
-  if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
-    rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
-
-    # move that build order to the end of the build-list
-    sed -i '
-      /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
-        $ b
-        d
-      }
-      $ a '"$1 $2 $3 $4" \
-      "${work_dir}/build-list"
-  fi
-
   # release lock on build-list - otherwise seed-build-list won't run
   flock -u 9
 
@@ -517,24 +497,9 @@ for package in ${packages}; do
   remove_old_package_versions 'i686' "${destination}" "${package}"
 done
 
-# remove old state files (these should be only "done" markers, but
-# actually we don't care what it is) - as long as it's not "testing" or "tested"
-find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
-  -not -name '*.testing' \
-  -not -name '*.tested' \
-  -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \
-  -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \
-  -exec rm '{}' \;
-
 # remove package from build list
 sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
 
-# remove package lock file
-if ! [ "${destination}" = 'build-support' ]; then
-  # shellcheck disable=SC2086
-  printf '%s\n' ${packages} > \
-    "${work_dir}/package-states/$1.$2.$3.$4.done"
-fi
 # shellcheck disable=SC2016
 {
   printf 'UPDATE `build_assignments`'
@@ -578,6 +543,3 @@ fi
   printf 'DROP TABLE `loops_to_delete`;\n'
 } | \
   mysql_run_query
-rm -f \
-  "${work_dir}/package-states/$1.$2.$3.$4.locked" \
-  "${work_dir}/package-states/$1.$2.$3.$4.broken"
diff --git a/lib/common-functions b/lib/common-functions
index 0a5ef1c..f66ccd1 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -4,10 +4,6 @@
 
 # shellcheck disable=SC2039
 
-# TODO: have full information (currently in files) in database
-
-# TODO: remove state files
-
 if [ -z "${base_dir}" ]; then
   # just to make shellcheck happy
   . 'conf/default.conf'
-- 
cgit v1.2.3-54-g00ecf


From 3edfb5d8d42567807b949ddac48efbb680ed4d9c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:21:22 +0100
Subject: bin/build-master-status-from-mysql: do not check consistency of
 database to build-list

---
 bin/build-master-status-from-mysql | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index 081baec..c6df53b 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -20,23 +20,6 @@ fi
 # shellcheck disable=SC2119
 mysql_cleanup
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`'
-  printf ' FROM `package_sources`'
-  mysql_join_package_sources_upstream_repositories
-  mysql_join_package_sources_build_assignments
-  mysql_join_build_assignments_binary_packages
-  mysql_join_binary_packages_repositories
-  printf ' WHERE `repositories`.`name`="build-list"'
-} | \
-  mysql_run_query | \
-  tr '\t' ' ' | \
-  sort > \
-  "${tmp_dir}/build-list.mysql"
-sort "${work_dir}/build-list" > \
-  "${tmp_dir}/build-list.file"
-
 # shellcheck disable=SC2016
 {
   printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
@@ -55,10 +38,6 @@ sort "${work_dir}/deletion-list" > \
 
 {
   mysql_sanity_check || true
-  diff -u \
-    "${tmp_dir}/build-list.file" \
-    "${tmp_dir}/build-list.mysql" || \
-    true
   diff -u \
     "${tmp_dir}/deletion-list.file" \
     "${tmp_dir}/deletion-list.mysql" || \
-- 
cgit v1.2.3-54-g00ecf


From d22efbc22053a482c6d1b9f264b12d54f509508c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:30:45 +0100
Subject: bin/sanity-check: read more information from the database

---
 bin/sanity-check | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/bin/sanity-check b/bin/sanity-check
index 4398595..e6b4184 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -145,20 +145,19 @@ while [ $# -gt 0 ]; do
         "${tmp_dir}/messages"
 
       errors=$(
-        grep -vn '^\S\+ [0-9a-f]\{40\} [0-9a-f]\{40\} \S\+$' "${work_dir}/build-list"
-      ) || true
-      if [ -n "${errors}" ]; then
-        if [ ${silence} -le 1 ]; then
-          printf '\nThe following build orders are wrongly formatted:\n%s\n' \
-            "${errors}" >> \
-            "${tmp_dir}/messages"
-        fi
-        i_am_insane
-      fi
-
-      errors=$(
-        cut -d' ' -f1 < \
-          "${work_dir}/build-list" | \
+        # shellcheck disable=SC2016
+        {
+          printf 'SELECT `architectures`.`name`,`package_sources`.`pkgbase` FROM `package_sources`'
+          mysql_join_package_sources_build_assignments
+          mysql_join_build_assignments_architectures
+          printf ' WHERE EXISTS('
+            printf 'SELECT * FROM `binary_packages`'
+            mysql_join_binary_packages_repositories
+            printf ' WHERE `repositories`.`name`="build-list"'
+            printf ' AND `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+          printf ');\n'
+        } | \
+          mysql_run_query | \
           sort | \
           uniq -d
       )
@@ -172,13 +171,16 @@ while [ $# -gt 0 ]; do
       fi
 
       errors=$(
+        # shellcheck disable=SC2016
         {
-          cut -d' ' -f1 < \
-            "${work_dir}/build-list"
-          cat "${work_dir}/deletion-list"
+          printf 'SELECT `a`.`pkgname` FROM `binary_packages` AS `a`'
+          mysql_join_binary_packages_repositories 'a' 'a_r'
+          printf ' AND `a_r`.`name`="build-list"'
+          printf ' JOIN `binary_packages` AS `b` ON `a`.`pkgname`=`b`.`pkgname`'
+          mysql_join_binary_packages_repositories 'b' 'b_r'
+          printf ' AND `b_r`.`name`="deletion-list";\n'
         } | \
-          sort | \
-          uniq -d
+          mysql_run_query
       )
       if [ -n "${errors}" ]; then
         if [ ${silence} -le 1 ]; then
-- 
cgit v1.2.3-54-g00ecf


From b3023cfe9c2d4e8319885e22a575bd1a522b3f6c Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:42:11 +0100
Subject: decommission build-list file

---
 bin/bootstrap-mysql       |  6 ------
 bin/get-assignment        |  9 ---------
 bin/get-package-updates   | 33 ++++-----------------------------
 bin/prioritize-build-list |  6 ------
 bin/return-assignment     |  3 ---
 bin/seed-build-list       |  2 --
 conf/default.conf         |  1 -
 7 files changed, 4 insertions(+), 56 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index fc347d7..817989d 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -796,12 +796,6 @@ fi
 } | \
   mysql_run_query
 
-tr ' ' '.' < \
-  "${work_dir}/build-list" | \
-  while read -r state_file; do
-    mysql_generate_package_metadata 'build-list' "${state_file}"
-  done
-
 grep '^\('"$(
   # shellcheck disable=SC2086
   printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories}
diff --git a/bin/get-assignment b/bin/get-assignment
index a82e1f8..7ac749f 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -20,15 +20,6 @@
 
 hand_out_assignment() {
 
-  # move that build order to the end of the build-list
-  sed -i '
-    /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
-      $ b
-      d
-    }
-    $ a '"$1 $2 $3 $4" \
-    "${work_dir}/build-list"
-
   # shellcheck disable=SC2016
   {
     printf 'SELECT '
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 3e9edf9..dcc9f3b 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -110,7 +110,6 @@ delete_package() {
   } | \
     mysql_run_query >> \
     "${work_dir}/deletion-list.new"
-  sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
   # TODO: Once we want to rely on the database for test_exclusion, we
   # need to run the command below unconditionally, but with some
   # changes, so we can easily revert.
@@ -201,9 +200,6 @@ echo 'Check modified packages from the last update, and put them to the build li
 # If a package is deleted, remove from the rebuild list, and add it to the deletion list.
 # If a new package is added, then ensure that it's not on the deletion list.
 
-cp \
-  "${work_dir}/build-list" \
-  "${work_dir}/build-list.new"
 cp \
   "${work_dir}/deletion-list" \
   "${work_dir}/deletion-list.new"
@@ -223,26 +219,9 @@ for repo in ${repo_names}; do
       git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}"
     fi
   } | \
-    # Packages which are already on the build list should receive a git_revision bump if _any_ file changed.
-    # Thus, we rename any file "PKGBUILD" to trigger the successive logic.
-    if [ "${repo}" = 'archlinux32' ]; then
-      sed "$(
-        sed '
-          s/ .*$//
-          s|^|\\@^.\\t[^/]\\+/|
-          s|$|/@ s@/[^/]*$@/PKGBUILD@|
-        ' "${work_dir}/build-list"
-      )"
-    else
-      sed "$(
-        sed '
-          s/ .*$//
-          s|^|\\@^.\\t|
-          s|$|/@ s@/[^/]*$@/PKGBUILD@|
-        ' "${work_dir}/build-list"
-      )"
-    fi | \
     # only track changes in PKGBUILDs
+    # TODO: However, packages which are already on the build list should
+    # receive a git_revision bump if _any_ file changed.
     grep '/PKGBUILD$' | \
     if [ "${repo}" = "archlinux32" ]; then
     # modify the directory structure from the modifiaction-repository
@@ -312,11 +291,6 @@ done | \
       fi
     fi
     if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
-      # new or modified PKGBUILD
-      sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
-      # shellcheck disable=SC2154
-      echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
-        "${work_dir}/build-list.new"
       sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new"
       # shellcheck disable=SC2016
       {
@@ -346,6 +320,7 @@ done | \
           )"
       } | \
         mysql_run_query
+      # shellcheck disable=SC2154
       mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       continue
     fi
@@ -530,7 +505,7 @@ mysql_find_build_assignment_loops
 # Move the .new-files to the actual files
 
 {
-  printf '%s\n' "build-list" "deletion-list"
+  printf '%s\n' "deletion-list"
   # shellcheck disable=SC2086
   printf '%s.revision\n' ${repo_names}
 } | \
diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list
index 1ff3d5a..db828ae 100755
--- a/bin/prioritize-build-list
+++ b/bin/prioritize-build-list
@@ -23,12 +23,6 @@ if ! flock -s -n 8; then
   exit 1
 fi
 
-{
-  sed -n "/^$1/p" "${work_dir}/build-list"
-  sed "/^$1/d" "${work_dir}/build-list"
-} | \
-  sponge "${work_dir}/build-list"
-
 # shellcheck disable=SC2016
 {
   printf 'UPDATE `build_assignments`'
diff --git a/bin/return-assignment b/bin/return-assignment
index da9e762..a2669f6 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -497,9 +497,6 @@ for package in ${packages}; do
   remove_old_package_versions 'i686' "${destination}" "${package}"
 done
 
-# remove package from build list
-sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
-
 # shellcheck disable=SC2016
 {
   printf 'UPDATE `build_assignments`'
diff --git a/bin/seed-build-list b/bin/seed-build-list
index 41707dd..ae08d57 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -374,8 +374,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \
         sed 's/ $/\n/'
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
-      printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \
-        "${work_dir}/build-list"
       printf '\n' >&2
     done
 
diff --git a/conf/default.conf b/conf/default.conf
index b508deb..e8e0169 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -133,7 +133,6 @@ if [ -z "${package_database_lock_file}" ]; then
 fi
 
 mkdir -p "${work_dir}"
-touch "${work_dir}/build-list"
 touch "${work_dir}/deletion-list"
 
 for repo in ${repo_names}; do
-- 
cgit v1.2.3-54-g00ecf


From ba6f43ca62a719d075cd23ccba35dbbab5f43951 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:45:47 +0100
Subject: decommission deletion-list

---
 bin/build-master-status-from-mysql | 20 --------------------
 bin/cleanup                        |  7 -------
 bin/get-package-updates            | 35 ++++-------------------------------
 conf/default.conf                  |  1 -
 4 files changed, 4 insertions(+), 59 deletions(-)

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index c6df53b..962b12e 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -20,28 +20,8 @@ fi
 # shellcheck disable=SC2119
 mysql_cleanup
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
-  printf ' FROM `package_sources`'
-  mysql_join_package_sources_build_assignments
-  mysql_join_build_assignments_binary_packages
-  mysql_join_binary_packages_repositories
-  printf ' WHERE `repositories`.`name`="deletion-list"'
-  printf ' OR `binary_packages`.`is_to_be_deleted`;\n'
-} | \
-  mysql_run_query | \
-  sort > \
-  "${tmp_dir}/deletion-list.mysql"
-sort "${work_dir}/deletion-list" > \
-  "${tmp_dir}/deletion-list.file"
-
 {
   mysql_sanity_check || true
-  diff -u \
-    "${tmp_dir}/deletion-list.file" \
-    "${tmp_dir}/deletion-list.mysql" || \
-    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
diff --git a/bin/cleanup b/bin/cleanup
index 7aaa39f..c9a8d2b 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -69,11 +69,4 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \
   -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \
   -delete
 
-# remove dependency graphs of packages on the deletion list
-sed '
-  s|^|'"${webserver_directory}"'/graphs/|
-  s|$|.png|
-' "${work_dir}/deletion-list" | \
-  xargs -rn1 rm -f
-
 exit 0
diff --git a/bin/get-package-updates b/bin/get-package-updates
index dcc9f3b..f4c53b4 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -99,17 +99,6 @@ fi
 # delete_package package
 # mark $package for deletion
 delete_package() {
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
-    printf ' FROM `package_sources`'
-    mysql_join_package_sources_build_assignments
-    mysql_join_build_assignments_binary_packages
-    printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
-      "$(printf '%s' "$1" | base64 -w0)"
-  } | \
-    mysql_run_query >> \
-    "${work_dir}/deletion-list.new"
   # TODO: Once we want to rely on the database for test_exclusion, we
   # need to run the command below unconditionally, but with some
   # changes, so we can easily revert.
@@ -200,10 +189,6 @@ echo 'Check modified packages from the last update, and put them to the build li
 # If a package is deleted, remove from the rebuild list, and add it to the deletion list.
 # If a new package is added, then ensure that it's not on the deletion list.
 
-cp \
-  "${work_dir}/deletion-list" \
-  "${work_dir}/deletion-list.new"
-
 for repo in ${repo_names}; do
   eval repo_path='"${repo_paths__'"${repo}"'}"'
   eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
@@ -291,7 +276,6 @@ done | \
       fi
     fi
     if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
-      sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new"
       # shellcheck disable=SC2016
       {
         # delete old build assignment and associated binary packages
@@ -329,9 +313,6 @@ done | \
     exit 1
   done
 
-sort -u "${work_dir}/deletion-list.new" | \
-  sponge "${work_dir}/deletion-list.new"
-
 echo 'apply blacklisting'
 # ignore blacklisted packages and dependent packages
 # this is the first time when all the information is available and up to date
@@ -462,15 +443,10 @@ echo "${black_listed}" | \
     delete_package "${package}"
   done
 
-sort -u "${work_dir}/deletion-list.new" | \
-  sponge "${work_dir}/deletion-list.new"
-
 
 if [ -n "${test_exclusion}" ]; then
-  # we should not actually update the build-list et. al, but solely print this difference:
-  if diff --color -u "${work_dir}/deletion-list" "${work_dir}/deletion-list.new"; then
-    printf 'If you put "%s" on the blacklist, no additional packages will end up on the deletion list.\n' "${test_exclusion}"
-  fi
+  # TODO: reimplement test_exclusion with information from the database
+  >&2 echo 'sry, not yet done.'
   exit 0
 fi
 
@@ -504,11 +480,8 @@ mysql_find_build_assignment_loops
 
 # Move the .new-files to the actual files
 
-{
-  printf '%s\n' "deletion-list"
-  # shellcheck disable=SC2086
-  printf '%s.revision\n' ${repo_names}
-} | \
+# shellcheck disable=SC2086
+printf '%s.revision\n' ${repo_names} | \
   while read -r file; do
     mv "${work_dir}/${file}.new" "${work_dir}/${file}"
   done
diff --git a/conf/default.conf b/conf/default.conf
index e8e0169..11483a0 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -133,7 +133,6 @@ if [ -z "${package_database_lock_file}" ]; then
 fi
 
 mkdir -p "${work_dir}"
-touch "${work_dir}/deletion-list"
 
 for repo in ${repo_names}; do
 
-- 
cgit v1.2.3-54-g00ecf


From 6f7e155ea0dd7f0e5894ec133be313bbdb4589ba Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 09:14:32 +0100
Subject: web-scripts/statistics.php: add option to chose time axis differently

---
 web-scripts/statistics.php | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/web-scripts/statistics.php b/web-scripts/statistics.php
index 01bbb52..c919f93 100644
--- a/web-scripts/statistics.php
+++ b/web-scripts/statistics.php
@@ -1,5 +1,10 @@
 <?php
 
+if (isset($_GET["from"]))
+  $min_time="from_base64(\"" . base64_encode("-".$_GET["from"]) . "\")";
+else
+  $min_time="\"-7 00:00:00\"";
+
 $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
 if ($mysql->connect_error) {
   die("Connection failed: " . $mysql->connect_error);
@@ -19,7 +24,7 @@ if (! $result = $mysql -> query(
     "`statistics`.`blocked_tasks_count`," .
     "`statistics`.`next_tasks_count`" .
     "FROM `statistics` " .
-    "WHERE `statistics`.`date`>=ADDTIME(NOW(),\"-7 00:00:00\") " .
+    "WHERE `statistics`.`date`>=ADDTIME(NOW()," . $min_time . ") " .
     "ORDER BY `statistics`.`date`"
   ))
   die($mysql->error);
-- 
cgit v1.2.3-54-g00ecf


From b798f420fda3be00914a8ac7a5817b651a9858f4 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 09:48:51 +0100
Subject: bin/sanity-check: check git heads in database, not in files

---
 bin/sanity-check | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/bin/sanity-check b/bin/sanity-check
index e6b4184..7d1b2f0 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -119,16 +119,28 @@ while [ $# -gt 0 ]; do
       for repo in ${repo_names}; do
         eval 'repo_path="${repo_paths__'"${repo}"'}"'
         repo_revision=$(
-          cat "${work_dir}/${repo}.revision"
+          # shellcheck disable=SC2016
+          {
+            printf 'SELECT `git_repositories`.`head` FROM `git_repositories`'
+            printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
+              "$(printf '%s' "${repo}" | base64 -w0)"
+          } | \
+            mysql_run_query
         )
-        if ! git -C "${repo_path}" archive "${repo_revision}" -- | \
-          tar -t > /dev/null; then
+        if ! obj_type=$(git -C "${repo_path}" cat-file -t "${repo_revision}" 2>/dev/null); then
           if [ ${silence} -le 1 ]; then
-            printf '\nThe repository %s (%s) cannot archive the current revision %s.\n' \
+            printf '\nThe repository %s (%s) does not know the current revision %s.\n' \
               "${repo}" "${repo_path}" "${repo_revision}" >> \
               "${tmp_dir}/messages"
           fi
           i_am_insane
+        elif [ "${obj_type}" != 'commit' ]; then
+          if [ ${silence} -le 1 ]; then
+            printf '\nThe repository %s (%s) knows the current revision %s, but it is not a commit, but a %s.\n' \
+              "${repo}" "${repo_path}" "${repo_revision}" "${obj_type}" >> \
+              "${tmp_dir}/messages"
+          fi
+          i_am_insane
         fi
       done
 
-- 
cgit v1.2.3-54-g00ecf


From aa91222717dabe20642aeb8ddf9b2a127ba19f0f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 09:57:21 +0100
Subject: lib/common-functions: find_git_repository_to_package_repository read
 info from database instead of git

---
 lib/common-functions | 40 +++++++++++++++-------------------------
 1 file changed, 15 insertions(+), 25 deletions(-)

diff --git a/lib/common-functions b/lib/common-functions
index f66ccd1..8425e9a 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -88,35 +88,25 @@ find_repository_with_commit() {
 find_git_repository_to_package_repository() {
 
   local repository
-  local package_repository
-  local repo_path
-
-  package_repository="$1"
 
-  if [ "$1" = 'build-support' ]; then
-    echo 'packages'
+  repository=$(
+    # shellcheck disable=SC2016
+    {
+      printf 'SELECT `git_repositories`.`name` FROM `git_repositories`'
+      mysql_join_git_repositories_upstream_repositories
+      printf ' WHERE `upstream_repositories`.`name`=from_base64("%s");\n' \
+        "$(printf '%s' "$1" | base64 -w0)"
+    } | \
+      mysql_run_query
+  )
+  if [ -z "${repository}" ]; then
+    >&2 echo "can't find git repository with package repository '$1'"
+    exit 1
+  else
+    echo "${repository}"
     return 0
   fi
 
-  for repository in ${repo_names}; do
-    if [ "${repository}" = "archlinux32" ]; then
-      continue
-    fi
-    eval 'repo_path="${repo_paths__'"${repository}"'}"'
-    if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \
-      tar -t --wildcards '*/repos' | \
-      grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \
-      cut -d/ -f3 | \
-      sed 's|-[^-]\+$||' | \
-      sort -u | \
-      grep -qxF "${package_repository}"; then
-      echo "${repository}"
-      return 0
-    fi
-  done
-  >&2 echo "can't find git repository with package repository '$1'"
-  exit 1
-
 }
 
 # repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
-- 
cgit v1.2.3-54-g00ecf


From 68eb1424fa9d08c1c60fc3b1807a67d266fb66e5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 10:07:52 +0100
Subject: bin/bootstrap-mysql: new column `is_black_listed` in
 `build_assignments`

---
 bin/bootstrap-mysql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 817989d..acf2170 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -88,6 +88,7 @@ if [ ! "$1" = 'slim' ]; then
       ' package_source BIGINT :package_sources' \
       ' architecture SMALLINT :architectures' \
       ' is_blocked VARCHAR(128) NULL' \
+      ' is_black_listed VARCHAR(128) NULL' \
       ' is_broken BIT' \
       ' priority SMALLINT' \
       ' UNIQUE content package_source architecture' \
-- 
cgit v1.2.3-54-g00ecf


From 3682408492801e1325f742267394db7cca7f2339 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 10:27:23 +0100
Subject: bin/bootstrap-mysql: build_assignments.is_black_listed should be TEXT

---
 bin/bootstrap-mysql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index acf2170..98f8efb 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -88,7 +88,7 @@ if [ ! "$1" = 'slim' ]; then
       ' package_source BIGINT :package_sources' \
       ' architecture SMALLINT :architectures' \
       ' is_blocked VARCHAR(128) NULL' \
-      ' is_black_listed VARCHAR(128) NULL' \
+      ' is_black_listed TEXT NULL' \
       ' is_broken BIT' \
       ' priority SMALLINT' \
       ' UNIQUE content package_source architecture' \
-- 
cgit v1.2.3-54-g00ecf


From a3888e3b4c363d7159e0274e75ecc7c7a9f3be1b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 10:37:08 +0100
Subject: bin/get-package-updates: save black-list(-reason) in database

---
 bin/get-package-updates | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index f4c53b4..1b01087 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -454,6 +454,30 @@ echo 'Done - mark decisions as final.'
 
 # shellcheck disable=SC2016
 {
+  # save blacklist into database
+  printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64), `reason` TEXT);\n'
+  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
+    tar -Ox 'blacklist' | \
+    sed -n '
+      s/^\(\S\+\)\s*#\s*/\1 /
+      T
+      p
+    ' | \
+    while read -r pkgbase reason; do
+      printf '(from_base64("%s"),from_base64("%s")),\n' \
+        "$(printf '%s' "${pkgbase}" | base64 -w0)" \
+        "$(printf '%s' "${reason}" | base64 -w0)"
+    done | \
+    sed '
+      1 i INSERT INTO `bl` (`pkgbase`,`reason`) VALUES
+      $ s/,$/;/
+    '
+  printf 'UPDATE `build_assignments`'
+  mysql_join_build_assignments_package_sources
+  printf ' LEFT JOIN `bl` ON `package_sources`.`pkgbase`=`bl`.`pkgbase`'
+  printf ' SET `build_assignments`.`is_black_listed`=`bl`.`reason`;\n'
+  printf 'DROP TEMPORARY TABLE `bl`;\n'
+
   # update hashes of repositories in mysql database
   for repo in ${repo_names}; do
     printf 'UPDATE `git_repositories`'
-- 
cgit v1.2.3-54-g00ecf


From 4e2dbce045fa380a19d76d861c6d17654c10b786 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 10:47:48 +0100
Subject: web-scripts/blacklist.php new

---
 web-scripts/blacklist.php | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 web-scripts/blacklist.php

diff --git a/web-scripts/blacklist.php b/web-scripts/blacklist.php
new file mode 100644
index 0000000..6ad46f4
--- /dev/null
+++ b/web-scripts/blacklist.php
@@ -0,0 +1,47 @@
+<html>
+<head>
+<title>Blacklisted packages</title>
+<link rel="stylesheet" type="text/css" href="/static/style.css">
+</head>
+<body>
+<table>
+<tr><th>architecture</th><th>package</th><th>reason</th></tr>
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+if ( ! $result = $mysql -> query(
+  "SELECT DISTINCT `architectures`.`name` AS `architecture`,`package_sources`.`pkgbase`,`build_assignments`.`is_black_listed` " .
+  "FROM `build_assignments` " .
+  "JOIN `architectures` ON `build_assignments`.`architecture`=`architectures`.`id` " .
+  "JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id` " .
+  "WHERE `build_assignments`.`is_black_listed` IS NOT NULL")) {
+  die($mysql->error);
+}
+if ($result -> num_rows > 0) {
+  while($row = $result->fetch_assoc()) {
+    print "<tr><td>";
+    print $row["architecture"];
+    print "</td><td>";
+    print $row["pkgbase"];
+    print "</td><td>";
+    print preg_replace(
+      array (
+        "/FS32#(\\d+)/",
+        "/FS#(\\d+)/"
+      ),
+      array (
+        "<a href=\"https://bugs.archlinux32.org/index.php?do=details&task_id=$1\">$0</a>",
+        "<a href=\"https://bugs.archlinux.org/task/$1\">$0</a>"
+      ),
+      $row["is_black_listed"]
+    );
+    print "</td></tr>\n";
+  }
+}
+?>
+</table>
+</body>
+</html>
-- 
cgit v1.2.3-54-g00ecf


From 3211e6e8de91e6cb462a1935032ffa209470f550 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 10:52:44 +0100
Subject: bin/build-master-status: blacklist.html is obsolete

---
 bin/build-master-status | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 18a5d5b..c93a9aa 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -190,42 +190,6 @@ if [ -s "${tmp_dir}/todos" ]; then
   rm -f "${tmp_dir}/todos"
 fi
 
-{
-  printf '%s\n' \
-    '<html>' \
-    '<head>' \
-    '<title>Blacklisted packages</title>' \
-    '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
-    '</head>' \
-    '<body>' \
-    '<table>'
-  printf '<tr>'
-  printf '<th>%s</th>' \
-    'package' \
-    'reason'
-  printf '</tr>\n'
-  git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
-    tar -Ox | \
-    sed '
-      s@FS#\([0-9]\+\)@<a href="https://bugs.archlinux.org/task/\1">\0</a>@
-      s@FS32#\([0-9]\+\)@<a href="https://bugs.archlinux32.org/index.php?do=details\&task_id=\1">\0</a>@
-      /.#/!s/$/#/
-      s|\(.\)#|\1</td><td>|
-      /^\s*#/{
-        s/^\s*#\s*//
-        s|\s*\(</td><td>\)|</font></s>\1|
-        s/^/<s><font color="#808080">/
-      }
-      s|^|<tr><td>|
-      s|$|</td></tr>|
-    '
-  printf '%s\n' \
-    '</table>' \
-    '</body>' \
-    '</html>'
-} > \
-  "${tmp_dir}/blacklist.html"
-
 {
   printf '%s\n' \
     '<html>' \
-- 
cgit v1.2.3-54-g00ecf


From 865de4ebc3753e78cd1fe2083d213c364afe4000 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 10:57:13 +0100
Subject: web-scripts/blacklist.php: order by pkgbase

---
 web-scripts/blacklist.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/web-scripts/blacklist.php b/web-scripts/blacklist.php
index 6ad46f4..c2810c6 100644
--- a/web-scripts/blacklist.php
+++ b/web-scripts/blacklist.php
@@ -17,7 +17,8 @@ if ( ! $result = $mysql -> query(
   "FROM `build_assignments` " .
   "JOIN `architectures` ON `build_assignments`.`architecture`=`architectures`.`id` " .
   "JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id` " .
-  "WHERE `build_assignments`.`is_black_listed` IS NOT NULL")) {
+  "WHERE `build_assignments`.`is_black_listed` IS NOT NULL " .
+  "ORDER BY `package_sources`.`pkgbase`")) {
   die($mysql->error);
 }
 if ($result -> num_rows > 0) {
-- 
cgit v1.2.3-54-g00ecf


From 5474aa6f94f4d83c4f3b03e00357b7fd47a8f2b8 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 10:59:53 +0100
Subject: bin/bootstrap-mysql: set head on bootstrapping to HEAD

---
 bin/bootstrap-mysql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 98f8efb..86fa645 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -622,8 +622,8 @@ fi
             base64 -w0
         )" \
         "$(
-          base64_encode_each < \
-            "${work_dir}/${repo}.revision"
+          git -C "${repo_path}" rev-parse HEAD | \
+            base64_encode_each
         )" | \
         sed 's|,$|),|'
     done
-- 
cgit v1.2.3-54-g00ecf


From 9d7f5d9f22c633381612bb2a1fbc036e54b40254 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 11:19:58 +0100
Subject: bin/get-package-updates: use revisions from database instead of files

---
 bin/get-package-updates | 70 +++++++++++++++++++++++--------------------------
 1 file changed, 33 insertions(+), 37 deletions(-)

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1b01087..cf2ea6c 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -127,31 +127,30 @@ delete_package() {
   fi
 }
 
-# Update git repositories (official packages, community packages and the repository of package customizations).
+something_new=false
 
 for repo in ${repo_names}; do
   eval repo_path='"${repo_paths__'"${repo}"'}"'
+  # Update git repositories (official packages, community packages and the repository of package customizations).
   git -C "${repo_path}" fetch origin master:master
-done
-
-# Read previous git revision numbers from files.
-
-something_new=false
-
-for repo in ${repo_names}; do
+  # read previous git revision numbers from database.
+  # shellcheck disable=SC2016
   eval "old_repo_revisions__${repo}='$(
-    cat "${work_dir}/${repo}.revision" 2> /dev/null || \
-      echo NONE
+    {
+      printf 'SELECT `git_repositories`.`head` FROM `git_repositories`'
+      printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
+        "$(printf '%s' "${repo}" | base64 -w0)"
+    } | \
+      mysql_run_query
   )'"
-  eval repo_path='"${repo_paths__'"${repo}"'}"'
-  eval "new_repo_revisions__${repo}='$(
-    if ${pull}; then
+  # determine new git revision
+  if ${pull}; then
+    eval "new_repo_revisions__${repo}='$(
       git -C "${repo_path}" rev-parse HEAD
-    else
-      cat "${work_dir}/${repo}.revision"
-    fi | \
-      tee "${work_dir}/${repo}.revision.new"
-  )'"
+    )'"
+  else
+    eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"'
+  fi
   if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then
     something_new=true
   fi
@@ -241,24 +240,26 @@ done | \
   while read -r mode package git_revision repository; do
     if [ "${mode}" = 'D' ]; then
       # deleted PKGBUILD
-      git_revision=$(cat "${work_dir}/archlinux32.revision.new")
+      # shellcheck disable=SC2154
+      git_revision="${new_repo_revisions__archlinux32}"
       found_package=false
       for repository in ${repo_names}; do
         eval 'repo_path="${repo_paths__'"${repository}"'}"'
         if [ "${repository}" = "archlinux32" ]; then
-          if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" 2> /dev/null | \
+          if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \
             tar -t 2> /dev/null | \
             grep -q "/$(str_to_regex "${package}")/PKGBUILD$"; then
             found_package=true
             break;
           fi
         else
-          if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" -- "${package}/repos" 2> /dev/null | \
-            tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \
-            cut -d/ -f3 | \
-            grep -v 'staging\|testing\|-unstable' | \
-            grep -vq -- '-i686$'; then
-            git_revision=$(cat "${work_dir}/${repository}.revision.new")
+          # shellcheck disable=SC2154
+          if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos" 2> /dev/null | ' \
+            'tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | ' \
+            'cut -d/ -f3 | ' \
+            'grep -v '"'"'staging\|testing\|-unstable'"'"' | ' \
+            'grep -vq -- '"'"'-i686$'"'"; then
+            eval 'git_revision="${new_repo_revisions__'"${repository}"'}"'
             found_package=true
             break;
           fi
@@ -420,11 +421,14 @@ if [ "${deletion_list_count}" -gt 1000 ]; then
     {
       printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
       for repo_name in ${repo_names}; do
-        if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then
+        eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"'
+        eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"'
+        # shellcheck disable=SC2154
+        if [ "${old_revision}" != "${new_revision}" ]; then
           printf '%s: %s -> %s\n' \
             "${repo_name}" \
-            "$(cat "${work_dir}/${repo_name}.revision")" \
-            "$(cat "${work_dir}/${repo_name}.revision.new")"
+            "${old_revision}" \
+            "${new_revision}"
         fi
       done
     } | \
@@ -502,14 +506,6 @@ echo 'Done - mark decisions as final.'
 # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list".
 mysql_find_build_assignment_loops
 
-# Move the .new-files to the actual files
-
-# shellcheck disable=SC2086
-printf '%s.revision\n' ${repo_names} | \
-  while read -r file; do
-    mv "${work_dir}/${file}.new" "${work_dir}/${file}"
-  done
-
 # Remove the lock file
 
 rm -f "${build_list_lock_file}"
-- 
cgit v1.2.3-54-g00ecf


From d90db40944ab8818c6207872fb8ae81fa2488056 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 15:11:54 +0100
Subject: bin/return-assignment: fix regex matching haskell packages

---
 bin/return-assignment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/return-assignment b/bin/return-assignment
index a2669f6..89a1f85 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -176,7 +176,7 @@ if [ "$5" = 'ERROR' ]; then
         ' | \
         tr ' ' '\n' | \
         sed '
-          s/^/-p ^\\(haskell-\\)\\?/
+          s/^/-p ^(haskell-)?/
           s/-[0-9.]\+$/\$/
         ' | \
         sort -u
-- 
cgit v1.2.3-54-g00ecf


From 497779257683e1c4ee2f2bf4c25687b34323c6be Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 23 Mar 2018 20:06:55 +0100
Subject: bin/check-opcodes: fix find expression to retrieve libraries only

---
 bin/check-opcodes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/check-opcodes b/bin/check-opcodes
index c7ab4a9..bd96cc6 100755
--- a/bin/check-opcodes
+++ b/bin/check-opcodes
@@ -101,7 +101,7 @@ log "Checking for architecture: $ARCH ($OPCODE_ARGS)"
 bsdtar --no-fflags -x -C $tmp_dir -f $PACKAGE
 
 # shellcheck disable=SC2044
-for absfile in $(find $tmp_dir -name '*.so*' -type f); do
+for absfile in $(find $tmp_dir -regextype grep -regex '.*\.so\(\.[0-9.]\+\)\?' -type f); do
 	file=$(basename $absfile)
 	log "Checking shared library: $file"
 	readelf -a $absfile > $tmp_dir/$file.elf
-- 
cgit v1.2.3-54-g00ecf