#!/bin/sh # report about status of build master # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" # TODO: replace by build-master-status-from-mysql usage() { >&2 echo '' >&2 echo 'build-master-status: report about status of build master' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -w|--web:' >&2 echo ' Output to webserver instead of stdout.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( getopt -o hw \ --long help \ --long web \ -n "$(basename "$0")" -- "$@" || \ echo usage )" web=false while true do case "$1" in -h|--help) usage 0 ;; -w|--web) web=true ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done if [ $# -ne 0 ]; then >&2 echo 'Too many arguments.' usage fi if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit fi tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT stable=$( ls_master_mirror 'i686' | \ grep -v 'testing$\|staging$\|-unstable$' | \ while read -r dir; do ls_master_mirror "i686/${dir}" done | \ grep -c '\.pkg\.tar\.xz$' ) tasks=$( grep -c '^\S\+ \S\+ \S\+ \S\+$' \ "${work_dir}/build-list" ) || true pending_packages=$( grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \ tr ' ' '.' | \ while read -r package; do generate_package_metadata "${package}" 2>&1 > /dev/null cat "${work_dir}/package-infos/${package}.packages" done | wc -l ) next_tasks=$( { cat "${work_dir}/build-list" find "${work_dir}/package-states" -maxdepth 1 \ \( -name '*.broken' -o -name '*.blocked' \) \ -printf '%f\n' | \ sed ' s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| p ' } | \ sort | \ uniq -u | \ while read -r package git_revision mod_git_revision repository; do if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" fi done | \ wc -l ) staging=$( find "${work_dir}/package-states" -name '*.done' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) testing=$( find "${work_dir}/package-states" -name '*.testing' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) tested=$( find "${work_dir}/package-states" -name '*.tested' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) { find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \ while read -r pkg rev mod_rev repo; do if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then echo "${pkg}" fi done { find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ -regex '.*/loop_[0-9]\+' \ -exec cat '{}' \; | \ sort -u find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\(\.[^.]\+\)\{4\}||' | \ sort -u } | \ sort | \ uniq -d } | \ sort -u > \ "${tmp_dir}/broken-packages-names" broken=$( wc -l < \ "${tmp_dir}/broken-packages-names" ) blocked=$( find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \ wc -l ) locked=$( find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \ wc -l ) loops=$( find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ -regex '.*/loop_[0-9]\+' | \ wc -l ) looped_packages=$( find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ -regex '.*/loop_[0-9]\+' \ -exec cat '{}' \; | \ sort -u | \ wc -l ) { printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \ "${stable}" \ "$((staging+testing+tested+pending_packages))" printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \ "$((tasks-broken))" \ "${tasks}" \ "${pending_packages}" \ "${next_tasks}" printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \ "$((testing+tested))" \ "${tested}" \ "${staging}" printf 'There are %d broken package builds.\n' \ "${broken}" if [ "${loops}" -ne 0 ]; then printf 'There are %d loops containing %d package builds.\n' \ "${loops}" \ "${looped_packages}" fi if [ $((broken+testing+tested+staging)) -ne 0 ]; then printf '%.1f%% of all packages are broken.\n' \ "$( echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \ bc )" fi if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then printf '%.1f%% of the planned work has been done.\n' \ "$( echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \ bc )" fi } > \ "${tmp_dir}/build-master-status.html" if ${web}; then "${base_dir}/bin/calculate-dependent-packages" { printf '%s\n' \ '' \ '' \ 'Status of archlinux32 build master' \ '' \ '' \ '' sed 's|$|
|' "${tmp_dir}/build-master-status.html" printf '%s\n' \ '
' \ 'currently building packages:
' \ '' printf '' printf '' \ 'since (UTC)' \ 'pkgname' \ 'git revision' \ 'modification git revision' \ 'package repository' \ 'build slave' printf '' find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \ -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \ -execdir sed ' :a $!{ N s/\n/, / ba } ' '{}' \; | \ sort -k1n,1 | \ sed ' s|^\S\+ || s|\.locked | | s|\.\([^.]\+\)$| \1| s|\.\([^.]\+\)$| \1| s|\.\([^.]\+\)$| \1| ' | \ while read -r date time pkg rev mod_rev repo slaves; do printf '' printf '' \ "${date} ${time}" \ "${pkg}" \ "

${rev}

" \ "

$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")

" \ "${repo}" \ "${slaves}" printf '\n' done printf '%s\n' \ '
%s
%s
' \ '' \ '' } | \ sponge "${tmp_dir}/build-master-status.html" end=$(($(date +%s)-7*24*60*60)) { [ -f "${webserver_directory}/statistics" ] && \ cat "${webserver_directory}/statistics" printf '%s ' \ "$(date +%s)" \ "${stable}" \ "${tasks}" \ "${pending_packages}" \ "${staging}" \ "${testing}" \ "${broken}" \ "${loops}" \ "${looped_packages}" \ "${locked}" \ "${blocked}" \ "${next_tasks}" \ "${tested}" | \ sed 's| $|\n|' echo "${end}" } | \ sort -k1nr,1 | \ sed -n " /^${end}\$/q p " | \ tac > \ "${tmp_dir}/statistics" find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \ -printf '%f build()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \ -printf '%f check()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \ -printf '%f prepare()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \ -printf '%f package()\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \ -printf '%f source\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \ -printf '%f dependencies\n' \ \) -o \ \( \ -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \ -printf '%f package-cache\n' \ \) -o \ \( \ -exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \ -printf '%f run-as-root\n' \ \) -o \ -printf '%f unknown\n' \ \) | \ sed ' s|\(\.[^.]\+\)\{3\} | | ' | \ sort -u | \ sed ' :a $!N s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/ ta P D ' | \ sort -k1,1 > \ "${tmp_dir}/broken-packages.reason" { printf '%s\n' \ '' \ '' \ 'List of broken package builds' \ '' \ '' \ '' \ 'build logs
' \ '' \ '' printf '' \ 'package' \ 'git revision' \ 'modification git revision' \ 'package repository' \ 'compilations' \ 'dependent' \ 'build error' \ 'blocked' printf '\n' find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\.broken$||' | \ sort -k1,1 | \ join -j 1 - "${tmp_dir}/broken-packages.reason" | \ sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \ while read -r sf pkg rev mod_rev repo build_error; do if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then printf '1 ' else printf '0 ' fi printf '%s ' \ "${pkg}" \ "${rev}" \ "${mod_rev}" \ "${repo}" \ "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \ "$( # shellcheck disable=SC2010 ls -t "${webserver_directory}/build-logs/error" | \ grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$' )" \ "$( { grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \ echo 'x  ' } | \ cut -d' ' -f2 )" \ "${build_error}" if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then sed ' s|\s\(wait for \)|\n\1|g ' "${work_dir}/package-states/${sf}.blocked" | \ while read -r blocked_reason; do if echo "${blocked_reason}" | \ grep -q '^wait for '; then printf 'wait for ' echo "${blocked_reason}" | \ sed ' s|^wait for || s@\( and \| or \)@\n\1\n@ ' | \ while read -r reason; do if [ "FS#${reason#FS#}" = "${reason}" ]; then printf '%s' \ "${reason#FS#}" \ "${reason}" elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then printf '%s' \ "${reason#FS32#}" \ "${reason}" elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then printf '%s' \ "${reason}" \ "${reason}" elif [ "${reason% *}" != "${reason}" ]; then printf '%s' \ "${reason}" else printf '%s' \ "${reason}" fi if read -r operator; then printf ' %s ' "${operator}" fi done else echo "${blocked_reason}" fi done | \ tr '\n' ' ' else printf ' ' fi printf '\n' done | \ sort -k6n,6 | \ while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do if [ "${buildable}" -eq 0 ]; then left='(' right=')' else unset left unset right fi printf '' mod_rev=$( modification_revision_link "${mod_rev}" "${repo}" "${pkg}" ) build_error=$( echo "${build_error}" | \ sed 's|,|, |g' ) printf '' \ ''"${left}${pkg}${right}"'' \ "

${rev}

" \ "

${mod_rev}

" \ "${repo}" \ ''"${count}"'' \ "${dependent}" \ "${build_error}" \ "${reason}" printf '\n' done printf '%s\n' \ '
%s
%s
' \ '' \ '' } > \ "${tmp_dir}/broken-packages.html" rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason" { printf '%s\n' \ '' \ '' \ 'Todos in the build scripts' \ '' \ '' find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ -exec grep -nHF '' '{}' \; | \ awk ' { print $0 } /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} ' | \ sed -n ' s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ T N s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ :a N s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ ta s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ p ' | \ tee "${tmp_dir}/todos" | \ sed ' :a N /\n$/!ba s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|TODO #\2 - \1 (line \3):\n| ' | \ sed ' s|$|
| ' printf '%s\n' \ '' \ '' } > \ "${tmp_dir}/todos.html" if [ -s "${tmp_dir}/todos" ]; then sed ' :a N /\n$/!ba s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 | s/\n$// s/\n/\\n/g ' -i "${tmp_dir}/todos" while read -r file line desc; do printf '%s %s %s\n' \ "$(printf '%s' "${file}" | base64 -w0)" \ "$(printf '%s' "${line}" | base64 -w0)" \ "$(printf '%s' "${desc}" | base64 -w0)" done < \ "${tmp_dir}/todos" | \ sponge "${tmp_dir}/todos" # update todos # shellcheck disable=SC2016 while read -r file line desc; do printf 'UPDATE IGNORE `todos`' printf ' SET `todos`.`line`=from_base64("%s")' \ "${line}" printf ' WHERE `todos`.`file`=from_base64("%s")' \ "${file}" printf ' AND `todos`.`description`=from_base64("%s");\n' \ "${desc}" printf 'UPDATE IGNORE `todos`' printf ' SET `todos`.`description`=from_base64("%s")' \ "${desc}" printf ' WHERE `todos`.`file`=from_base64("%s")' \ "${file}" printf ' AND `todos`.`line`=from_base64("%s");\n' \ "${line}" done < \ "${tmp_dir}/todos" | \ mysql_run_query # insert unfound todos # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ mysql_run_query --raw --batch | \ sed ' 1d 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' printf ';\n' printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' while read -r file line desc; do printf '(' printf 'from_base64("%s"),' \ "${file}" \ "${line}" \ "${desc}" | \ sed 's/,$/),/' done < \ "${tmp_dir}/todos" | \ sed ' s/,$// ' printf ';\n' printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) ' printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` ' printf 'FROM `td` ' printf 'WHERE NOT EXISTS (' printf 'SELECT * FROM `todos`' printf ' AND `td`.`%s`=`todos`.`%s`' \ 'file' 'file' \ 'line' 'line' \ 'description' 'description' | \ sed 's/^ AND / WHERE /' printf ');\n' printf 'DELETE FROM `todos` WHERE NOT EXISTS (' printf 'SELECT * FROM `td`' printf ' AND `td`.`%s`=`todos`.`%s`' \ 'file' 'file' \ 'line' 'line' \ 'description' 'description' | \ sed 's/^ AND / WHERE /' printf ');' printf 'DROP TABLE `td`;\n' printf 'DELETE FROM `todo_links` WHERE NOT EXISTS (' printf 'SELECT * FROM `todos` ' printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`' printf ') OR NOT EXISTS (' printf 'SELECT * FROM `todos` ' printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' printf ');\n' } | \ mysql_run_query rm -f "${tmp_dir}/todos" fi { printf '%s\n' \ '' \ '' \ 'Blacklisted packages' \ '' \ '' \ '' \ '' printf '' printf '' \ 'package' \ 'reason' printf '\n' git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \ tar -Ox | \ sed ' s@FS#\([0-9]\+\)@\0@ s@FS32#\([0-9]\+\)@\0@ /.#/!s/$/#/ s|\(.\)#|\1| ' printf '%s\n' \ '
%s
| /^\s*#/{ s/^\s*#\s*// s|\s*\(\)|\1| s/^// } s|^|
| s|$|
' \ '' \ '' } > \ "${tmp_dir}/blacklist.html" { printf '%s\n' \ '' \ '' \ 'log of ssh connections from build slaves' \ '' \ '' \ '' printf '' printf '' \ 'time' \ 'build slave' \ 'command' \ 'arguments' printf '\n' if [ -r "${work_dir}/ssh-log" ]; then tac "${work_dir}/ssh-log" | \ while read -r date time slave command arguments; do printf '' printf '' \ "${date} ${time}" \ "${slave}" \ "${command}" \ "${arguments}" printf '\n' done fi printf '%s\n' \ '
%s
%s
' \ '' \ '' } > \ "${tmp_dir}/ssh-log.html" find "${tmp_dir}" -maxdepth 1 -type f | \ while read -r file; do cat "${file}" > \ "${webserver_directory}/${file##*/}" done else cat "${tmp_dir}/build-master-status.html" fi