#!/bin/sh # report about status of build master # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" usage() { >&2 echo '' >&2 echo 'build-master-status: report about status of build master' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -w|--web:' >&2 echo ' Output to webserver instead of stdout.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( getopt -o hw \ --long help \ --long web \ -n "$(basename "$0")" -- "$@" || \ echo usage )" web=false while true do case "$1" in -h|--help) usage 0 ;; -w|--web) web=true ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done if [ $# -ne 0 ]; then >&2 echo 'Too many arguments.' usage fi if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit fi tmp_dir=$(mktemp -d) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT stable=$( ls_master_mirror 'i686' | \ grep -v 'testing$\|staging$\|-unstable$' | \ while read -r dir; do ls_master_mirror "i686/${dir}" done | \ grep -c '\.pkg\.tar\.xz$' ) tasks=$( grep -c '^\S\+ \S\+ \S\+ \S\+$' \ "${work_dir}/build-list" ) || true pending_packages=$( grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \ tr ' ' '.' | \ while read -r package; do generate_package_metadata "${package}" 2>&1 > /dev/null cat "${work_dir}/package-infos/${package}.packages" done | wc -l ) next_tasks=$( { cat "${work_dir}/build-list" find "${work_dir}/package-states" -maxdepth 1 \ \( -name '*.broken' -o -name '*.blocked' \) \ -printf '%f\n' | \ sed ' s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| p ' } | \ sort | \ uniq -u | \ while read -r package git_revision mod_git_revision repository; do if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" fi done | \ wc -l ) staging=$( find "${work_dir}/package-states" -name '*.done' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) testing=$( find "${work_dir}/package-states" -name '*.testing' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) tested=$( find "${work_dir}/package-states" -name '*.tested' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) { find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \ while read -r pkg rev mod_rev repo; do if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then echo "${pkg}" fi done { find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ -regex '.*/loop_[0-9]\+' \ -exec cat '{}' \; | \ sort -u find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\(\.[^.]\+\)\{4\}||' | \ sort -u } | \ sort | \ uniq -d } | \ sort -u > \ "${tmp_dir}/broken-packages-names" broken=$( wc -l < \ "${tmp_dir}/broken-packages-names" ) blocked=$( find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \ wc -l ) locked=$( find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \ wc -l ) loops=$( find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ -regex '.*/loop_[0-9]\+' | \ wc -l ) looped_packages=$( find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ -regex '.*/loop_[0-9]\+' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) { printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \ "${stable}" \ "$((staging+testing+tested+pending_packages))" printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \ "$((tasks-broken))" \ "${tasks}" \ "${pending_packages}" \ "${next_tasks}" printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \ "$((testing+tested))" \ "${tested}" \ "${staging}" printf 'There are %d broken package builds.\n' \ "${broken}" if [ "${loops}" -ne 0 ]; then printf 'There are %d loops containing %d package builds.\n' \ "${loops}" \ "${looped_packages}" fi if [ $((broken+testing+tested+staging)) -ne 0 ]; then printf '%.1f%% of all packages are broken.\n' \ "$( echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \ bc )" fi if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then printf '%.1f%% of the planned work has been done.\n' \ "$( echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \ bc )" fi } > \ "${tmp_dir}/build-master-status.html" if ${web}; then "${base_dir}/bin/calculate-dependent-packages" { printf '%s\n' \ '' \ '' \ 'Status of archlinux32 build master' \ '' \ '' \ '' sed 's|$|
|' "${tmp_dir}/build-master-status.html" printf '%s\n' \ '
' \ 'currently building packages:
' \ '' printf '' printf '' \ 'since (UTC)' \ 'pkgname' \ 'git revision' \ 'modification git revision' \ 'package repository' \ 'build slave' printf '' find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \ -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \ -execdir sed ' :a $!{ N s/\n/, / ba } ' '{}' \; | \ sort -k1n,1 | \ sed ' s|^\S\+ || s|\.locked | | s|\.\([^.]\+\)$| \1| s|\.\([^.]\+\)$| \1| s|\.\([^.]\+\)$| \1| ' | \ while read -r date time pkg rev mod_rev repo slaves; do printf '' printf '' \ "${date} ${time}" \ "${pkg}" \ "

${rev}

" \ "

$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")

" \ "${repo}" \ "${slaves}" printf '\n' done printf '%s\n' \ '
%s
%s
' \ '' \ '' } | \ sponge "${tmp_dir}/build-master-status.html" end=$(($(date +%s)-7*24*60*60)) { [ -f "${webserver_directory}/statistics" ] && \ cat "${webserver_directory}/statistics" printf '%s ' \ "$(date +%s)" \ "${stable}" \ "${tasks}" \ "${pending_packages}" \ "${staging}" \ "${testing}" \ "${broken}" \ "${loops}" \ "${looped_packages}" \ "${locked}" \ "${blocked}" \ "${next_tasks}" \ "${tested}" | \ sed 's| $|\n|' echo "${end}" } | \ sort -k1nr,1 | \ sed -n " /^${end}\$/q p " | \ tac > \ "${tmp_dir}/statistics" find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \ -printf '%f build()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \ -printf '%f check()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \ -printf '%f prepare()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in package()\.$' {} \; \ -printf '%f package()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \ -printf '%f source\n' \ \) -o \ \( \ -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \ -printf '%f package-cache\n' \ \) -o \ -printf '%f unknown\n' \ \) | \ sed ' s|\(\.[^.]\+\)\{3\} | | ' | \ sort -u | \ sed ' :a $!N s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/ ta P D ' | \ sort -k1,1 > \ "${tmp_dir}/broken-packages.reason" { printf '%s\n' \ '' \ '' \ 'List of broken package builds' \ '' \ '' \ '' \ 'build logs
' \ '' \ '' printf '' \ 'package' \ 'git revision' \ 'modification git revision' \ 'package repository' \ 'compilations' \ 'dependent' \ 'build error' \ 'blocked' printf '\n' find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\.broken$||' | \ sort -k1,1 | \ join -j 1 - "${tmp_dir}/broken-packages.reason" | \ sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \ while read -r sf pkg rev mod_rev repo build_error; do if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then printf '1 ' else printf '0 ' fi printf '%s ' \ "${pkg}" \ "${rev}" \ "${mod_rev}" \ "${repo}" \ "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \ "$( # shellcheck disable=SC2010 ls -t "${webserver_directory}/build-logs/error" | \ grep -m1 "^$(str_to_regex "${sf}.")[^.]\+\.build-log\.gz\$" )" \ "$( { grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \ echo 'x  ' } | \ cut -d' ' -f2 )" \ "${build_error}" if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then sed ' s|\s\(wait for \)|\n\1|g ' "${work_dir}/package-states/${sf}.blocked" | \ while read -r blocked_reason; do if echo "${blocked_reason}" | \ grep -q '^wait for '; then printf 'wait for ' echo "${blocked_reason}" | \ sed ' s|^wait for || s@\( and \| or \)@\n\1\n@ ' | \ while read -r reason; do if [ "FS#${reason#FS#}" = "${reason}" ]; then printf '%s' \ "${reason#FS#}" \ "${reason}" elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then printf '%s' \ "${reason#FS32#}" \ "${reason}" elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then printf '%s' \ "${reason}" \ "${reason}" elif [ "${reason% *}" != "${reason}" ]; then printf '%s' \ "${reason}" else printf '%s' \ "${reason}" fi if read -r operator; then printf ' %s ' "${operator}" fi done else echo "${blocked_reason}" fi done | \ tr '\n' ' ' else printf ' ' fi printf '\n' done | \ sort -k6n,6 | \ while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do if [ "${buildable}" -eq 0 ]; then left='(' right=')' else unset left unset right fi printf '' mod_rev=$( modification_revision_link "${mod_rev}" "${repo}" "${pkg}" ) build_error=$( echo "${build_error}" | \ sed 's|,|, |g' ) printf '' \ ''"${left}${pkg}${right}"'' \ "

${rev}

" \ "

${mod_rev}

" \ "${repo}" \ ''"${count}"'' \ "${dependent}" \ "${build_error}" \ "${reason}" printf '\n' done printf '%s\n' \ '
%s
%s
' \ '' \ '' } > \ "${tmp_dir}/broken-packages.html" rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason" { printf '%s\n' \ '' \ '' \ 'Todos in the build scripts' \ '' \ '' find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ -exec grep -nHF '' '{}' \; | \ awk ' { print $0 } /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} ' | \ sed -n ' s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ T N s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ :a N s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ ta s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ p ' | \ sed ' :a N /\n$/!ba s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|TODO #\2 - \1 (line \3):\n| ' | \ sed ' s|$|
| ' printf '%s\n' \ '' \ '' } > \ "${tmp_dir}/todos.html" { printf '%s\n' \ '' \ '' \ 'Blacklisted packages' \ '' \ '' \ '' \ '' printf '' printf '' \ 'package' \ 'reason' printf '\n' git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \ tar -Ox | \ sed ' s@FS#\([0-9]\+\)@\0@ s@FS32#\([0-9]\+\)@\0@ /.#/!s/$/#/ s|\(.\)#|\1| ' printf '%s\n' \ '
%s
| /^\s*#/{ s/^\s*#\s*// s|\s*\(\)|\1| s/^// } s|^|
| s|$|
' \ '' \ '' } > \ "${tmp_dir}/blacklist.html" { printf '%s\n' \ '' \ '' \ 'log of ssh connections from build slaves' \ '' \ '' \ '' printf '' printf '' \ 'time' \ 'build slave' \ 'command' \ 'arguments' printf '\n' if [ -r "${work_dir}/ssh-log" ]; then tac "${work_dir}/ssh-log" | \ while read -r date time slave command arguments; do printf '' printf '' \ "${date} ${time}" \ "${slave}" \ "${command}" \ "${arguments}" printf '\n' done fi printf '%s\n' \ '
%s
%s
' \ '' \ '' } > \ "${tmp_dir}/ssh-log.html" find "${tmp_dir}" -maxdepth 1 -type f | \ while read -r file; do cat "${file}" > \ "${webserver_directory}/${file##*/}" done else cat "${tmp_dir}/build-master-status.html" fi