#!/bin/sh # report about status of build master # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" usage() { >&2 echo '' >&2 echo 'build-master-status: report about status of build master' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -w|--web:' >&2 echo ' Output to webserver instead of stdout.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( getopt -o hw \ --long help \ --long web \ -n "$(basename "$0")" -- "$@" || \ echo usage )" web=false while true do case "$1" in -h|--help) usage 0 ;; -w|--web) web=true ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done if [ $# -ne 0 ]; then >&2 echo 'Too many arguments.' usage fi tmp_dir=$(mktemp -d) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT stable=$( ls_master_mirror 'i686' | \ grep -v 'testing$\|staging$\|-unstable$' | \ while read -r dir; do ls_master_mirror "i686/${dir}" done | \ grep -c '\.pkg\.tar\.xz$' ) tasks=$( grep -c '^\S\+ \S\+ \S\+ \S\+$' \ "${work_dir}/build-list" ) || true pending_packages=$( grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \ tr ' ' '.' | \ while read -r package; do generate_package_metadata "${package}" 2>&1 > /dev/null cat "${work_dir}/package-infos/${package}.packages" done | wc -l ) next_tasks=$( ( grep -vxF 'break_loops' "${work_dir}/build-list" find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed ' s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| p ' ) | \ sort | \ uniq -u | \ while read -r package git_revision mod_git_revision repository; do if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" fi done | \ wc -l ) staging=$( ls_master_mirror 'i686' | \ grep 'staging$' | \ while read -r dir; do ls_master_mirror "i686/${dir}" done | \ grep -c '\.pkg\.tar\.xz$' ) testing=$( ls_master_mirror 'i686' | \ grep 'testing$' | \ while read -r dir; do ls_master_mirror "i686/${dir}" done | \ grep -c '\.pkg\.tar\.xz$' ) ( find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \ while read -r pkg rev mod_rev repo; do if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then echo "${pkg}" fi done ( find "${work_dir}/build-list.loops" -maxdepth 1 | \ grep '/loop_[0-9]\+$' | \ xargs -r cat | \ sort -u find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\(\.[^.]\+\)\{4\}||' | \ sort -u ) | \ sort | \ uniq -d ) | \ sort -u > \ "${tmp_dir}/broken-packages-names" broken=$( wc -l < \ "${tmp_dir}/broken-packages-names" ) blocked=$( find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \ wc -l ) locked=$( find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \ wc -l ) loops=$( find "${work_dir}/build-list.loops" -maxdepth 1 | \ grep -c '/loop_[0-9]\+$' ) || true looped_packages=$( find "${work_dir}/build-list.loops" -maxdepth 1 | \ grep '/loop_[0-9]\+$' | \ xargs -r cat | \ sort -u | \ wc -l ) { printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \ "${stable}" \ "$((staging+testing+pending_packages))" printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \ "$((tasks-broken))" \ "${tasks}" \ "${pending_packages}" \ "${next_tasks}" printf 'There are %d testing and %d staging packages.\n' \ "${testing}" \ "${staging}" printf 'There are %d broken package builds.\n' \ "${broken}" if [ "${loops}" -ne 0 ]; then printf 'There are %d loops containing %d package builds.\n' \ "${loops}" \ "${looped_packages}" fi if [ $((broken+testing+staging)) -ne 0 ]; then printf '%.1f%% of all packages are broken.\n' \ "$( echo "scale=10; 100*${broken}/(${broken}+${testing}+${staging})" | \ bc )" fi if [ $((testing+staging+pending_packages-broken)) -ne 0 ]; then printf '%.1f%% of the planned work has been done.\n' \ "$( echo "scale=10; 100*(${testing}+${staging})/(${testing}+${staging}+${pending_packages}-${broken})" | \ bc )" fi } > \ "${tmp_dir}/build-master-status.html" if ${web}; then "${base_dir}/bin/calculate-dependent-packages" ( printf '\n\nStatus of archlinux32 build master\n\n\n' sed 's|$|
|' "${tmp_dir}/build-master-status.html" printf '\n' ) | \ sponge "${tmp_dir}/build-master-status.html" end=$(($(date +%s)-7*24*60*60)) ( [ -f "${webserver_directory}/statistics" ] && \ cat "${webserver_directory}/statistics" printf '%s ' \ "$(date +%s)" \ "${stable}" \ "${tasks}" \ "${pending_packages}" \ "${staging}" \ "${testing}" \ "${broken}" \ "${loops}" \ "${looped_packages}" \ "${locked}" \ "${blocked}" \ "${next_tasks}" | \ sed 's| $|\n|' echo "${end}" ) | \ sort -k1nr,1 | \ sed -n " /^${end}\$/q p " | \ tac > \ "${tmp_dir}/statistics" { printf '\n\nList of broken package builds\n\n\nbuild logs
\n\n' printf '' \ 'package' \ 'git revision' \ 'modification git revision' \ 'package repository' \ 'compilations' \ 'dependent' \ 'blocked' printf '\n' find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ sed 's|\.broken$||' | \ sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\)$|\1 \2 \3 \4 \5|' | \ sort -k1,1 | \ join -1 2 -2 1 -o 1.1,1.2,1.3,1.4,1.5 - "${tmp_dir}/broken-packages-names" | \ while read -r sf pkg rev mod_rev repo; do printf '%s ' \ "${pkg}" \ "${rev}" \ "${mod_rev}" \ "${repo}" \ "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \ "$( ls -t "${webserver_directory}/build-logs" | \ grep -m1 "^$(str_to_regex "${sf}.")[^.]\+\.build-log\.gz\$" )" \ "$( ( grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \ echo 'x  ' ) | \ cut -d' ' -f2 )" if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then tr '\n' ' ' < "${work_dir}/package-states/${sf}.blocked" | \ sed 's|FS#\([0-9]\+\)|FS#\1|' else printf ' ' fi printf '\n' done | \ sort -k5n,5 | \ while read -r pkg rev mod_rev repo count log_file dependent reason; do printf '' printf '' \ ''"${pkg}"'' \ "${rev}" \ "${mod_rev}" \ "${repo}" \ ''"${count}"'' \ "${dependent}" \ "${reason}" printf '\n' done printf '
%s
%s
\n\n\n' } > \ "${tmp_dir}/broken-packages.html" rm -f "${tmp_dir}/broken-packages-names" find "${tmp_dir}" -maxdepth 1 | \ while read -r file; do cat "${file}" > \ "${webserver_directory}/${file##*/}" done else cat "${tmp_dir}/build-master-status.html" fi