path: root/bin/build-master-status

#!/bin/sh

# report about status of build master

# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"

# TODO: replace by build-master-status-from-mysql

usage() {
  >&2 echo ''
  >&2 echo 'build-master-status: report about status of build master'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -w|--web:'
  >&2 echo '    Output to webserver instead of stdout.'
  >&2 echo '  -h|--help:'
  >&2 echo '    Show this help and exit.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o hw \
    --long help \
    --long web \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

web=false

while true
do
  case "$1" in
    -h|--help)
      usage 0
    ;;
    -w|--web)
      web=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  exit
fi

tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT

stable=$(
  ls_master_mirror 'i686' | \
    grep -v 'testing$\|staging$\|-unstable$' | \
    while read -r dir; do
      ls_master_mirror "i686/${dir}"
    done | \
    grep -c '\.pkg\.tar\.xz$'
)
tasks=$(
  grep -c '^\S\+ \S\+ \S\+ \S\+$' \
    "${work_dir}/build-list"
) || true
pending_packages=$(
  grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \
    tr ' ' '.' | \
    while read -r package; do
      generate_package_metadata "${package}" 2>&1 > /dev/null
      cat "${work_dir}/package-infos/${package}.packages"
    done |
    wc -l
)
next_tasks=$(
  {
    cat "${work_dir}/build-list"
    find "${work_dir}/package-states" -maxdepth 1 \
      \( -name '*.broken' -o -name '*.blocked' \) \
      -printf '%f\n' | \
      sed '
        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
        p
      '
  } | \
    sort | \
    uniq -u | \
    while read -r package git_revision mod_git_revision repository; do
      if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
        echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
      fi
    done | \
    wc -l
)
staging=$(
  find "${work_dir}/package-states" -name '*.done' \
    -exec cat '{}' \; | \
    sort -u | \
    wc -l
)
testing=$(
  find "${work_dir}/package-states" -name '*.testing' \
    -exec cat '{}' \; | \
    sort -u | \
    wc -l
)
tested=$(
  find "${work_dir}/package-states" -name '*.tested' \
    -exec cat '{}' \; | \
    sort -u | \
    wc -l
)
{
  find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
    sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
    while read -r pkg rev mod_rev repo; do
      if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then
        echo "${pkg}"
      fi
    done
  {
    find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
      -regex '.*/loop_[0-9]\+' \
      -exec cat '{}' \; | \
      sort -u
    find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
      sed 's|\(\.[^.]\+\)\{4\}||' | \
      sort -u
  } | \
    sort | \
    uniq -d
} | \
  sort -u > \
  "${tmp_dir}/broken-packages-names"
broken=$(
  wc -l < \
    "${tmp_dir}/broken-packages-names"
)
blocked=$(
  find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \
    wc -l
)
locked=$(
  find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \
    wc -l
)
loops=$(
  find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
    -regex '.*/loop_[0-9]\+' | \
    wc -l
)
looped_packages=$(
  find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
    -regex '.*/loop_[0-9]\+' \
    -exec cat '{}' \; | \
    sort -u | \
    wc -l
)

{
  printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \
    "${stable}" \
    "$((staging+testing+tested+pending_packages))"
  printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \
    "$((tasks-broken))" \
    "${tasks}" \
    "${pending_packages}" \
    "${next_tasks}"
  printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \
    "$((testing+tested))" \
    "${tested}" \
    "${staging}"
  printf 'There are %d broken package builds.\n' \
    "${broken}"
  if [ "${loops}" -ne 0 ]; then
    printf 'There are %d loops containing %d package builds.\n' \
      "${loops}" \
      "${looped_packages}"
  fi
  if [ $((broken+testing+tested+staging)) -ne 0 ]; then
    printf '%.1f%% of all packages are broken.\n' \
      "$(
        echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \
          bc
      )"
  fi
  if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then
    printf '%.1f%% of the planned work has been done.\n' \
      "$(
        echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \
          bc
      )"
  fi
} > \
  "${tmp_dir}/build-master-status.html"

if ${web}; then
  "${base_dir}/bin/calculate-dependent-packages"
  {
    printf '%s\n' \
      '<html>' \
      '<head>' \
      '<title>Status of archlinux32 build master</title>' \
      '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
      '</head>' \
      '<body>'
    sed 's|$|<br>|' "${tmp_dir}/build-master-status.html"
    printf '%s\n' \
      '<br>' \
      'currently building packages:<br>' \
      '<table>'
    printf '<tr>'
    printf '<th>%s</th>' \
      'since (UTC)' \
      'pkgname' \
      'git revision' \
      'modification git revision' \
      'package repository' \
      'build slave'
    printf '</tr>'
    find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \
      -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \
      -execdir sed '
        :a
          $!{
            N
            s/\n/, /
            ba
          }
      ' '{}' \; | \
      sort -k1n,1 | \
      sed '
        s|^\S\+ ||
        s|\.locked | |
        s|\.\([^.]\+\)$| \1|
        s|\.\([^.]\+\)$| \1|
        s|\.\([^.]\+\)$| \1|
      ' | \
      while read -r date time pkg rev mod_rev repo slaves; do
        printf '<tr>'
        printf '<td>%s</td>' \
          "${date} ${time}" \
          "${pkg}" \
          "<p style=\"font-size:8px\">${rev}</p>" \
          "<p style=\"font-size:8px\">$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")</p>" \
          "${repo}" \
          "${slaves}"
        printf '</tr>\n'
      done
    printf '%s\n' \
      '</table>' \
      '</body>' \
      '</html>'
  } | \
    sponge "${tmp_dir}/build-master-status.html"
  end=$(($(date +%s)-7*24*60*60))
  {
    [ -f "${webserver_directory}/statistics" ] && \
      cat "${webserver_directory}/statistics"
    printf '%s ' \
      "$(date +%s)" \
      "${stable}" \
      "${tasks}" \
      "${pending_packages}" \
      "${staging}" \
      "${testing}" \
      "${broken}" \
      "${loops}" \
      "${looped_packages}" \
      "${locked}" \
      "${blocked}" \
      "${next_tasks}" \
      "${tested}" | \
      sed 's| $|\n|'
    echo "${end}"
  } | \
    sort -k1nr,1 | \
    sed -n "
      /^${end}\$/q
      p
    " | \
    tac > \
    "${tmp_dir}/statistics"

  find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \
    \( \
      -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \
      -printf '%f build()\n' \
    \) -o \
    \( \
      -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \
      -printf '%f check()\n' \
    \) -o \
    \( \
      -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \
      -printf '%f prepare()\n' \
    \) -o \
    \( \
      -exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \
      -printf '%f package()\n' \
    \) -o \
    \( \
      -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \
      -printf '%f source\n' \
    \) -o \
    \( \
      -exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \
      -printf '%f dependencies\n' \
    \) -o \
    \( \
      -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \
      -printf '%f package-cache\n' \
    \) -o \
    \( \
      -exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \
      -printf '%f run-as-root\n' \
    \) -o \
    -printf '%f unknown\n' \
    \) | \
    sed '
      s|\(\.[^.]\+\)\{3\} | |
    ' | \
    sort -u | \
    sed '
      :a
        $!N
        s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/
      ta
      P
      D
    ' | \
    sort -k1,1 > \
    "${tmp_dir}/broken-packages.reason"

  {
    printf '%s\n' \
      '<html>' \
      '<head>' \
      '<title>List of broken package builds</title>' \
      '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
      '</head>' \
      '<body>' \
      '<a href="build-logs/">build logs</a><br>' \
      '<table>' \
      '<tr>'
    printf '<th>%s</th>' \
      'package' \
      'git revision' \
      'modification git revision' \
      'package repository' \
      'compilations' \
      'dependent' \
      'build error' \
      'blocked'
    printf '</tr>\n'
    find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
      sed 's|\.broken$||' | \
      sort -k1,1 | \
      join -j 1 - "${tmp_dir}/broken-packages.reason" | \
      sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \
      while read -r sf pkg rev mod_rev repo build_error; do
        if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then
          printf '1 '
        else
          printf '0 '
        fi
        printf '%s ' \
          "${pkg}" \
          "${rev}" \
          "${mod_rev}" \
          "${repo}" \
          "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \
          "$(
            # shellcheck disable=SC2010
            ls -t "${webserver_directory}/build-logs/error" | \
              grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$'
          )" \
          "$(
            {
              grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \
                echo 'x &nbsp;'
            } | \
              cut -d' ' -f2
          )" \
          "${build_error}"
        if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then
          sed '
            s|\s\(wait for \)|\n\1|g
          ' "${work_dir}/package-states/${sf}.blocked" | \
            while read -r blocked_reason; do
              if echo "${blocked_reason}" | \
                grep -q '^wait for '; then
                printf 'wait for '
                echo "${blocked_reason}" | \
                  sed '
                    s|^wait for ||
                    s@\( and \| or \)@\n\1\n@
                  ' | \
                  while read -r reason; do
                    if [ "FS#${reason#FS#}" = "${reason}" ]; then
                      printf '<a href="https://bugs.archlinux.org/task/%s">%s</a>' \
                        "${reason#FS#}" \
                        "${reason}"
                    elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then
                      printf '<a href="https://bugs.archlinux32.org/index.php?do=details&task_id=%s">%s</a>' \
                        "${reason#FS32#}" \
                        "${reason}"
                    elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then
                      printf '<a href="graphs/%s.png">%s</a>' \
                        "${reason}" \
                        "${reason}"
                    elif [ "${reason% *}" != "${reason}" ]; then
                      printf '%s' \
                        "${reason}"
                    else
                      printf '<font color="red">%s</font>' \
                        "${reason}"
                    fi
                    if read -r operator; then
                      printf ' %s ' "${operator}"
                    fi
                  done
              else
                echo "${blocked_reason}"
              fi
            done | \
              tr '\n' ' '
        else
          printf '&nbsp;'
        fi
        printf '\n'
      done | \
      sort -k6n,6 | \
      while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do
        if [ "${buildable}" -eq 0 ]; then
          left='('
          right=')'
        else
          unset left
          unset right
        fi
        printf '<tr>'
        mod_rev=$(
          modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
        )
        build_error=$(
          echo "${build_error}" | \
            sed 's|,|, |g'
        )
        printf '<td>%s</td>' \
          '<a href="graphs/'"${pkg}"'.png">'"${left}${pkg}${right}"'</a>' \
          "<p style=\"font-size:8px\">${rev}</p>" \
          "<p style=\"font-size:8px\">${mod_rev}</p>" \
          "${repo}" \
          '<a href="build-logs/error/'"${log_file}"'">'"${count}"'</a>' \
          "${dependent}" \
          "${build_error}" \
          "${reason}"
        printf '</tr>\n'
      done
    printf '%s\n' \
      '</table>' \
      '</body>' \
      '</html>'
  } > \
    "${tmp_dir}/broken-packages.html"

  rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason"

  {
    printf '%s\n' \
      '<html>' \
      '<head>' \
      '<title>Todos in the build scripts</title>' \
      '</head>' \
      '<body>'
    find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
      -exec grep -nHF '' '{}' \; | \
      awk '
        { print $0 }
        /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i}
      ' | \
      sed -n '
        s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/
        T
        N
        s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/
        :a
          N
          s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/
          ta
        s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
        p
      ' | \
      tee "${tmp_dir}/todos" | \
      sed '
        :a
        N
        /\n$/!ba
        s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n|
      ' | \
      sed '
        s|$|<br>|
      '
    printf '%s\n' \
      '</body>' \
      '</html>'
  } > \
    "${tmp_dir}/todos.html"

  if [ -s "${tmp_dir}/todos" ]; then
    sed '
      :a
      N
      /\n$/!ba
      s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 |
      s/\n$//
      s/\n/\\n/g
    ' -i "${tmp_dir}/todos"
    while read -r file line desc; do
      printf '%s %s %s\n' \
        "$(printf '%s' "${file}" | base64 -w0)" \
        "$(printf '%s' "${line}" | base64 -w0)" \
        "$(printf '%s' "${desc}" | base64 -w0)"
    done < \
      "${tmp_dir}/todos" | \
      sponge "${tmp_dir}/todos"
    # update todos
    # shellcheck disable=SC2016
    while read -r file line desc; do
      printf 'UPDATE `todos`'
      printf ' SET `todos`.`line`=from_base64("%s")' \
        "${line}"
      printf ' WHERE `todos`.`file`=from_base64("%s")' \
        "${file}"
      printf ' AND `todos`.`description`=from_base64("%s");\n' \
        "${desc}"

      printf 'UPDATE `todos`'
      printf ' SET `todos`.`description`=from_base64("%s")' \
        "${desc}"
      printf ' WHERE `todos`.`file`=from_base64("%s")' \
        "${file}"
      printf ' AND `todos`.`line`=from_base64("%s");\n' \
        "${line}"
    done < \
      "${tmp_dir}/todos" | \
      ${mysql_command}
    # insert unfound todos
    # shellcheck disable=SC2016
    while read -r file line desc; do
      printf ' ('
      printf 'from_base64("%s"),' \
        "${file}" \
        "${line}" \
        "${desc}" | \
        sed 's/,$/),/'
    done < \
      "${tmp_dir}/todos" | \
      sed '
        1 i INSERT IGNORE INTO `todos` (`file`,`line`,`description`) VALUES
        s/,$/;\n/
      ' | \
      ${mysql_command}
    rm -f "${tmp_dir}/todos"
  fi

  {
    printf '%s\n' \
      '<html>' \
      '<head>' \
      '<title>Blacklisted packages</title>' \
      '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
      '</head>' \
      '<body>' \
      '<table>'
    printf '<tr>'
    printf '<th>%s</th>' \
      'package' \
      'reason'
    printf '</tr>\n'
    git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
      tar -Ox | \
      sed '
        s@FS#\([0-9]\+\)@<a href="https://bugs.archlinux.org/task/\1">\0</a>@
        s@FS32#\([0-9]\+\)@<a href="https://bugs.archlinux32.org/index.php?do=details\&task_id=\1">\0</a>@
        /.#/!s/$/#/
        s|\(.\)#|\1</td><td>|
        /^\s*#/{
          s/^\s*#\s*//
          s|\s*\(</td><td>\)|</font></s>\1|
          s/^/<s><font color="#808080">/
        }
        s|^|<tr><td>|
        s|$|</td></tr>|
      '
    printf '%s\n' \
      '</table>' \
      '</body>' \
      '</html>'
  } > \
    "${tmp_dir}/blacklist.html"

  {
    printf '%s\n' \
      '<html>' \
      '<head>' \
      '<title>log of ssh connections from build slaves</title>' \
      '</head>' \
      '<body>' \
      '<table>'
    printf '<tr>'
    printf '<th>%s</th>' \
      'time' \
      'build slave' \
      'command' \
      'arguments'
    printf '</tr>\n'
    if [ -r "${work_dir}/ssh-log" ]; then
      tac "${work_dir}/ssh-log" | \
        while read -r date time slave command arguments; do
          printf '<tr>'
          printf '<td>%s</td>' \
            "${date} ${time}" \
            "${slave}" \
            "${command}" \
            "${arguments}"
          printf '</tr>\n'
        done
    fi
    printf '%s\n' \
      '</table>' \
      '</body>' \
      '</html>'
  } > \
    "${tmp_dir}/ssh-log.html"

  find "${tmp_dir}" -maxdepth 1 -type f | \
    while read -r file; do
      cat "${file}" > \
        "${webserver_directory}/${file##*/}"
    done

else
  cat "${tmp_dir}/build-master-status.html"
fi