bin/build-master-status + bin/build-master-status-from-mysql -> bin/build-master-status

2 files changed, 178 insertions, 276 deletions

diff --git a/bin/build-master-status b/bin/build-master-status
index 04e8ae8..9503d71 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -5,147 +5,76 @@
 # shellcheck source=../conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: replace by build-master-status-from-mysql
-
-usage() {
-  >&2 echo ''
-  >&2 echo 'build-master-status: report about status of build master'
-  >&2 echo ''
-  >&2 echo 'possible options:'
-  >&2 echo '  -h|--help:'
-  >&2 echo '    Show this help and exit.'
-  [ -z "$1" ] && exit 1 || exit "$1"
-}
-
-eval set -- "$(
-  getopt -o h \
-    --long help \
-    -n "$(basename "$0")" -- "$@" || \
-  echo usage
-)"
-
-while true
-do
-  case "$1" in
-    -h|--help)
-      usage 0
-    ;;
-    --)
-      shift
-      break
-    ;;
-    *)
-      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
-      exit 42
-    ;;
-  esac
-  shift
-done
-
-if [ $# -ne 0 ]; then
-  >&2 echo 'Too many arguments.'
-  usage
-fi
+tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-if [ -s "${work_dir}/build-master-sanity" ]; then
-  >&2 echo 'Build master is not sane.'
+# do not block if locked
+exec 9> "${sanity_check_lock_file}"
+if ! flock -n 9; then
+  >&2 echo 'Mysql-Sanity check skipped, cannot acquire lock.'
   exit
 fi
 
-tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+# shellcheck disable=SC2119
+mysql_cleanup
 
+# update todos
 find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
   -exec grep -nHF '' '{}' \; | \
-  awk '
-    { print $0 }
-    /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i}
-  ' | \
+  sed 's,^'"$(str_to_regex "${base_dir}")"'/,,' | \
   sed -n '
-    s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/
-    T
-    N
-    s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/
-    :a
-      N
-      s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/
-      ta
-    s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
-    p
+    /^[^:]\+:[0-9]\+:\s*#\s*TODO:/ {
+      :a
+      $! {
+        N
+        s/\n[^:[:space:]]\+:[0-9]\+:\s*\([^#[:space:]].*\)\?$//
+        Ta
+      }
+      s/\n[^:[:space:]]\+:[0-9]\+:\s*#\s*/\\\\n/g
+      s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\t\2\t/
+      p
+    }
   ' > "${tmp_dir}/todos"
 
 if [ -s "${tmp_dir}/todos" ]; then
-  sed '
-    :a
-    N
-    /\n$/!ba
-    s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 |
-    s/\n$//
-    s/\n/\\n/g
-  ' -i "${tmp_dir}/todos"
-  while read -r file line desc; do
-    printf '%s %s %s\n' \
-      "$(printf '%s' "${file}" | base64 -w0)" \
-      "$(printf '%s' "${line}" | base64 -w0)" \
-      "$(printf '%s' "${desc}" | base64 -w0)"
-  done < \
-    "${tmp_dir}/todos" | \
-    sponge "${tmp_dir}/todos"
-  # update todos
-  # shellcheck disable=SC2016
-  while read -r file line desc; do
-    printf 'UPDATE IGNORE `todos`'
-    printf ' SET `todos`.`line`=from_base64("%s")' \
-      "${line}"
-    printf ' WHERE `todos`.`file`=from_base64("%s")' \
-      "${file}"
-    printf ' AND `todos`.`description`=from_base64("%s");\n' \
-      "${desc}"
-
-    printf 'UPDATE IGNORE `todos`'
-    printf ' SET `todos`.`description`=from_base64("%s")' \
-      "${desc}"
-    printf ' WHERE `todos`.`file`=from_base64("%s")' \
-      "${file}"
-    printf ' AND `todos`.`line`=from_base64("%s");\n' \
-      "${line}"
-  done < \
-    "${tmp_dir}/todos" | \
-    mysql_run_query
-  # insert unfound todos
   # shellcheck disable=SC2016
   {
     printf 'SHOW CREATE TABLE `todos`' | \
       mysql_run_query | \
       sed '
         1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
+        s/ NOT NULL AUTO_INCREMENT/ NULL/
+        s/PRIMARY KEY/INDEX/
       '
     printf ';\n'
-    printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES '
-    while read -r file line desc; do
-      printf '('
-      printf 'from_base64("%s"),' \
-        "${file}" \
-        "${line}" \
-        "${desc}" | \
-        sed 's/,$/),/'
-    done < \
-      "${tmp_dir}/todos" | \
-      sed '
-        s/,$//
-      '
-    printf ';\n'
-    printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) '
-    printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` '
-    printf 'FROM `td` '
-    printf 'WHERE NOT EXISTS ('
-    printf 'SELECT * FROM `todos`'
-    printf ' AND `td`.`%s`=`todos`.`%s`' \
+    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `td` (`file`,`line`,`description`);\n' \
+      "${tmp_dir}/todos"
+
+    for matches in 'file line description' 'file description' 'file line' 'description'; do
+      printf 'UPDATE `td`'
+      printf ' JOIN `todos`'
+      for match in ${matches}; do
+        printf ' AND `td`.`%s`=`todos`.`%s`' \
+          "${match}" "${match}"
+      done | \
+        sed 's/^ AND / ON /'
+      printf ' SET `td`.`id`=`todos`.`id`'
+      printf ' WHERE `td`.`id` IS NULL;\n'
+    done
+
+    printf 'UPDATE `todos`'
+    printf ' JOIN `td` ON `todos`.`id`=`td`.`id`'
+    printf ',`todos`.`%s`=`td`.`%s`' \
       'file' 'file' \
       'line' 'line' \
       'description' 'description' | \
-      sed 's/^ AND / WHERE /'
-    printf ');\n'
+      sed 's/^,/ SET /'
+    printf ';\n'
+
+    printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`)'
+    printf ' SELECT `td`.`file`,`td`.`line`,`td`.`description`'
+    printf ' FROM `td`'
+    printf ' WHERE `td`.`id` IS NULL;\n'
 
     printf 'DELETE FROM `todos` WHERE NOT EXISTS ('
     printf 'SELECT * FROM `td`'
@@ -154,22 +83,140 @@ if [ -s "${tmp_dir}/todos" ]; then
       'line' 'line' \
       'description' 'description' | \
       sed 's/^ AND / WHERE /'
-    printf ');'
-    printf 'DROP TABLE `td`;\n'
+    printf ');\n'
+    printf 'DROP TEMPORARY TABLE `td`;\n'
     printf 'DELETE FROM `todo_links` WHERE NOT EXISTS ('
-    printf 'SELECT * FROM `todos` '
-    printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`'
+      printf 'SELECT * FROM `todos` '
+      printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`'
     printf ') OR NOT EXISTS ('
-    printf 'SELECT * FROM `todos` '
-    printf 'WHERE `todos`.`id`=`todo_links`.`dependent`'
+      printf 'SELECT * FROM `todos` '
+      printf 'WHERE `todos`.`id`=`todo_links`.`dependent`'
     printf ');\n'
   } | \
     mysql_run_query
-  rm -f "${tmp_dir}/todos"
+fi
+rm -f "${tmp_dir}/todos"
+
+{
+  mysql_sanity_check || true
+} | \
+  sed '
+    s,^-.*$,<font color="#FF0000">\0</font>,
+    s,^+.*$,<font color="#00FF00">\0</font>,
+    s/$/<br>/
+    1 i <html><head><title>sanity of the buildmaster'"'"'s mysql database</title></head><body>
+    $ a </body></html>
+  ' | \
+  sponge "${webserver_directory}/mysql-sanity.html"
+
+if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
+  [ ! -s "${work_dir}/build-master-sanity" ]; then
+  printf 'girls, my database is dirty again ...\n' | \
+    irc_say
+  echo 'build master is insane' > \
+    "${work_dir}/build-master-sanity"
 fi
 
-find "${tmp_dir}" -maxdepth 1 -type f | \
-  while read -r file; do
-    cat "${file}" > \
-      "${webserver_directory}/${file##*/}"
-  done
+if [ ! -s "${work_dir}/build-master-sanity" ]; then
+  # shellcheck disable=SC2016
+  {
+    printf 'INSERT IGNORE INTO `statistics` ('
+    printf '`%s`,' \
+      'stable_packages_count' \
+      'pending_tasks_count' \
+      'pending_packages_count' \
+      'staging_packages_count' \
+      'testing_packages_count' \
+      'tested_packages_count' \
+      'broken_tasks_count' \
+      'dependency_loops_count' \
+      'dependency_looped_tasks_count' \
+      'locked_tasks_count' \
+      'blocked_tasks_count' \
+      'next_tasks_count' | \
+      sed 's/,$//'
+    printf ') VALUES ('
+    # stable_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="stable"),'
+    # pending_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"),'
+    # pending_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"),'
+    # staging_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="staging"),'
+    # testing_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="testing"'
+    printf ' AND NOT `binary_packages`.`is_tested`),'
+    # tested_packages_count
+    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+    printf ' `binary_packages`'
+    mysql_join_binary_packages_repositories
+    mysql_join_repositories_repository_stabilities
+    printf ' WHERE `repository_stabilities`.`name`="testing"'
+    printf ' AND `binary_packages`.`is_tested`),'
+    # broken_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_assignments`.`is_broken`),'
+    # dependency_loops_count
+    printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM'
+    printf ' `build_dependency_loops`),'
+    # dependency_looped_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM'
+    printf ' `build_dependency_loops`),'
+    # locked_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM'
+    printf ' `build_slaves`'
+    mysql_join_build_slaves_build_assignments
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"),'
+    # blocked_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND `build_assignments`.`is_blocked` IS NOT NULL),'
+    # next_tasks_count
+    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+    printf ' `build_assignments`'
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_repositories
+    printf ' WHERE `repositories`.`name`="build-list"'
+    printf ' AND NOT EXISTS ('
+      printf 'SELECT * FROM `dependencies`'
+      mysql_join_dependencies_dependency_types
+      printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+      mysql_join_dependencies_install_target_providers
+      mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+      mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+      printf ' WHERE `prov_r`.`name`="build-list"'
+      printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`'
+    printf '))'
+    printf ');\n'
+  } | \
+    mysql_run_query
+fi
diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
deleted file mode 100755
index 4b00278..0000000
--- a/bin/build-master-status-from-mysql
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/bin/sh
-
-# report about status of build master - according to mysql database
-
-# shellcheck source=../conf/default.conf
-. "${0%/*}/../conf/default.conf"
-
-# TODO: add all (necessary) features from build-master-status
-
-tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-# do not block if locked
-exec 9> "${sanity_check_lock_file}"
-if ! flock -n 9; then
-  >&2 echo 'Mysql-Sanity check skipped, cannot acquire lock.'
-  exit
-fi
-
-# shellcheck disable=SC2119
-mysql_cleanup
-
-{
-  mysql_sanity_check || true
-} | \
-  sed '
-    s,^-.*$,<font color="#FF0000">\0</font>,
-    s,^+.*$,<font color="#00FF00">\0</font>,
-    s/$/<br>/
-    1 i <html><head><title>sanity of the buildmaster'"'"'s mysql database</title></head><body>
-    $ a </body></html>
-  ' | \
-  sponge "${webserver_directory}/mysql-sanity.html"
-
-if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
-  [ ! -s "${work_dir}/build-master-sanity" ]; then
-  printf 'girls, my database is dirty again ...\n' | \
-    irc_say
-  echo 'build master is insane' > \
-    "${work_dir}/build-master-sanity"
-fi
-
-if [ ! -s "${work_dir}/build-master-sanity" ]; then
-  # shellcheck disable=SC2016
-  {
-    printf 'INSERT IGNORE INTO `statistics` ('
-    printf '`%s`,' \
-      'stable_packages_count' \
-      'pending_tasks_count' \
-      'pending_packages_count' \
-      'staging_packages_count' \
-      'testing_packages_count' \
-      'tested_packages_count' \
-      'broken_tasks_count' \
-      'dependency_loops_count' \
-      'dependency_looped_tasks_count' \
-      'locked_tasks_count' \
-      'blocked_tasks_count' \
-      'next_tasks_count' | \
-      sed 's/,$//'
-    printf ') VALUES ('
-    # stable_packages_count
-    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="stable"),'
-    # pending_tasks_count
-    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"),'
-    # pending_packages_count
-    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"),'
-    # staging_packages_count
-    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="staging"),'
-    # testing_packages_count
-    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="testing"'
-    printf ' AND NOT `binary_packages`.`is_tested`),'
-    # tested_packages_count
-    printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
-    printf ' `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_repositories_repository_stabilities
-    printf ' WHERE `repository_stabilities`.`name`="testing"'
-    printf ' AND `binary_packages`.`is_tested`),'
-    # broken_tasks_count
-    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"'
-    printf ' AND `build_assignments`.`is_broken`),'
-    # dependency_loops_count
-    printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM'
-    printf ' `build_dependency_loops`),'
-    # dependency_looped_tasks_count
-    printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM'
-    printf ' `build_dependency_loops`),'
-    # locked_tasks_count
-    printf '(SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM'
-    printf ' `build_slaves`'
-    mysql_join_build_slaves_build_assignments
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"),'
-    # blocked_tasks_count
-    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"'
-    printf ' AND `build_assignments`.`is_blocked` IS NOT NULL),'
-    # next_tasks_count
-    printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
-    printf ' `build_assignments`'
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_repositories
-    printf ' WHERE `repositories`.`name`="build-list"'
-    printf ' AND NOT EXISTS ('
-      printf 'SELECT * FROM `dependencies`'
-      mysql_join_dependencies_dependency_types
-      printf ' AND `dependency_types`.`relevant_for_binary_packages`'
-      mysql_join_dependencies_install_target_providers
-      mysql_join_install_target_providers_binary_packages '' 'prov_bp'
-      mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
-      printf ' WHERE `prov_r`.`name`="build-list"'
-      printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`'
-    printf '))'
-    printf ');\n'
-  } | \
-    mysql_run_query
-fi