diff options
Diffstat (limited to 'bin/build-master-status')
| -rwxr-xr-x | bin/build-master-status | 695 |
1 files changed, 164 insertions, 531 deletions
diff --git a/bin/build-master-status b/bin/build-master-status index 4d47976..c93a9aa 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -5,37 +5,31 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: replace by build-master-status-from-mysql + usage() { >&2 echo '' >&2 echo 'build-master-status: report about status of build master' >&2 echo '' >&2 echo 'possible options:' - >&2 echo ' -w|--web:' - >&2 echo ' Output to webserver instead of stdout.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( - getopt -o hw \ + getopt -o h \ --long help \ - --long web \ -n "$(basename "$0")" -- "$@" || \ echo usage )" -web=false - while true do case "$1" in -h|--help) usage 0 ;; - -w|--web) - web=true - ;; --) shift break @@ -61,538 +55,177 @@ fi tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -stable=$( - ls_master_mirror 'i686' | \ - grep -v 'testing$\|staging$\|-unstable$' | \ - while read -r dir; do - ls_master_mirror "i686/${dir}" - done | \ - grep -c '\.pkg\.tar\.xz$' -) -tasks=$( - grep -c '^\S\+ \S\+ \S\+ \S\+$' \ - "${work_dir}/build-list" -) || true -pending_packages=$( - grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \ - tr ' ' '.' | \ - while read -r package; do - generate_package_metadata "${package}" 2>&1 > /dev/null - cat "${work_dir}/package-infos/${package}.packages" - done | - wc -l -) -next_tasks=$( - { - cat "${work_dir}/build-list" - find "${work_dir}/package-states" -maxdepth 1 \ - \( -name '*.broken' -o -name '*.blocked' \) \ - -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - p - ' - } | \ - sort | \ - uniq -u | \ - while read -r package git_revision mod_git_revision repository; do - if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then - echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - fi - done | \ - wc -l -) -staging=$( - find "${work_dir}/package-states" -name '*.done' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) -testing=$( - find "${work_dir}/package-states" -name '*.testing' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) -tested=$( - find "${work_dir}/package-states" -name '*.tested' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) { - find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \ - while read -r pkg rev mod_rev repo; do - if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then - echo "${pkg}" - fi - done - { - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat '{}' \; | \ - sort -u - find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}||' | \ - sort -u - } | \ - sort | \ - uniq -d -} | \ - sort -u > \ - "${tmp_dir}/broken-packages-names" -broken=$( - wc -l < \ - "${tmp_dir}/broken-packages-names" -) -blocked=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \ - wc -l -) -locked=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \ - wc -l -) -loops=$( - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' | \ - wc -l -) -looped_packages=$( - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) - -{ - printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \ - "${stable}" \ - "$((staging+testing+tested+pending_packages))" - printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \ - "$((tasks-broken))" \ - "${tasks}" \ - "${pending_packages}" \ - "${next_tasks}" - printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \ - "$((testing+tested))" \ - "${tested}" \ - "${staging}" - printf 'There are %d broken package builds.\n' \ - "${broken}" - if [ "${loops}" -ne 0 ]; then - printf 'There are %d loops containing %d package builds.\n' \ - "${loops}" \ - "${looped_packages}" - fi - if [ $((broken+testing+tested+staging)) -ne 0 ]; then - printf '%.1f%% of all packages are broken.\n' \ - "$( - echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \ - bc - )" - fi - if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then - printf '%.1f%% of the planned work has been done.\n' \ - "$( - echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \ - bc - )" - fi -} > \ - "${tmp_dir}/build-master-status.html" - -if ${web}; then - "${base_dir}/bin/calculate-dependent-packages" - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>Status of archlinux32 build master</title>' \ - '<link rel="stylesheet" type="text/css" href="/static/style.css">' \ - '</head>' \ - '<body>' - sed 's|$|<br>|' "${tmp_dir}/build-master-status.html" - printf '%s\n' \ - '<br>' \ - 'currently building packages:<br>' \ - '<table>' - printf '<tr>' - printf '<th>%s</th>' \ - 'since (UTC)' \ - 'pkgname' \ - 'git revision' \ - 'modification git revision' \ - 'package repository' \ - 'build slave' - printf '</tr>' - find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \ - -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \ - -execdir sed ' - :a - $!{ - N - s/\n/, / - ba - } - ' '{}' \; | \ - sort -k1n,1 | \ - sed ' - s|^\S\+ || - s|\.locked | | - s|\.\([^.]\+\)$| \1| - s|\.\([^.]\+\)$| \1| - s|\.\([^.]\+\)$| \1| - ' | \ - while read -r date time pkg rev mod_rev repo slaves; do - printf '<tr>' - printf '<td>%s</td>' \ - "${date} ${time}" \ - "${pkg}" \ - "<p style=\"font-size:8px\">${rev}</p>" \ - "<p style=\"font-size:8px\">$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")</p>" \ - "${repo}" \ - "${slaves}" - printf '</tr>\n' - done - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } | \ - sponge "${tmp_dir}/build-master-status.html" - end=$(($(date +%s)-7*24*60*60)) - { - [ -f "${webserver_directory}/statistics" ] && \ - cat "${webserver_directory}/statistics" - printf '%s ' \ - "$(date +%s)" \ - "${stable}" \ - "${tasks}" \ - "${pending_packages}" \ - "${staging}" \ - "${testing}" \ - "${broken}" \ - "${loops}" \ - "${looped_packages}" \ - "${locked}" \ - "${blocked}" \ - "${next_tasks}" \ - "${tested}" | \ - sed 's| $|\n|' - echo "${end}" - } | \ - sort -k1nr,1 | \ - sed -n " - /^${end}\$/q + printf '%s\n' \ + '<html>' \ + '<head>' \ + '<title>Todos in the build scripts</title>' \ + '</head>' \ + '<body>' + find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ + -exec grep -nHF '' '{}' \; | \ + awk ' + { print $0 } + /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} + ' | \ + sed -n ' + s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ + T + N + s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ + :a + N + s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ + ta + s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ p - " | \ - tac > \ - "${tmp_dir}/statistics" - - find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \ - -printf '%f build()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \ - -printf '%f check()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \ - -printf '%f prepare()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \ - -printf '%f package()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \ - -printf '%f source\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \ - -printf '%f dependencies\n' \ - \) -o \ - \( \ - -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \ - -printf '%f package-cache\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \ - -printf '%f run-as-root\n' \ - \) -o \ - -printf '%f unknown\n' \ - \) | \ - sed ' - s|\(\.[^.]\+\)\{3\} | | ' | \ - sort -u | \ + tee "${tmp_dir}/todos" | \ sed ' :a - $!N - s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/ - ta - P - D + N + /\n$/!ba + s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n| ' | \ - sort -k1,1 > \ - "${tmp_dir}/broken-packages.reason" - - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>List of broken package builds</title>' \ - '<link rel="stylesheet" type="text/css" href="/static/style.css">' \ - '</head>' \ - '<body>' \ - '<a href="build-logs/">build logs</a><br>' \ - '<table>' \ - '<tr>' - printf '<th>%s</th>' \ - 'package' \ - 'git revision' \ - 'modification git revision' \ - 'package repository' \ - 'compilations' \ - 'dependent' \ - 'build error' \ - 'blocked' - printf '</tr>\n' - find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\.broken$||' | \ - sort -k1,1 | \ - join -j 1 - "${tmp_dir}/broken-packages.reason" | \ - sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \ - while read -r sf pkg rev mod_rev repo build_error; do - if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then - printf '1 ' - else - printf '0 ' - fi - printf '%s ' \ - "${pkg}" \ - "${rev}" \ - "${mod_rev}" \ - "${repo}" \ - "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \ - "$( - # shellcheck disable=SC2010 - ls -t "${webserver_directory}/build-logs/error" | \ - grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$' - )" \ - "$( - { - grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \ - echo 'x ' - } | \ - cut -d' ' -f2 - )" \ - "${build_error}" - if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then - sed ' - s|\s\(wait for \)|\n\1|g - ' "${work_dir}/package-states/${sf}.blocked" | \ - while read -r blocked_reason; do - if echo "${blocked_reason}" | \ - grep -q '^wait for '; then - printf 'wait for ' - echo "${blocked_reason}" | \ - sed ' - s|^wait for || - s@\( and \| or \)@\n\1\n@ - ' | \ - while read -r reason; do - if [ "FS#${reason#FS#}" = "${reason}" ]; then - printf '<a href="https://bugs.archlinux.org/task/%s">%s</a>' \ - "${reason#FS#}" \ - "${reason}" - elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then - printf '<a href="https://bugs.archlinux32.org/index.php?do=details&task_id=%s">%s</a>' \ - "${reason#FS32#}" \ - "${reason}" - elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then - printf '<a href="graphs/%s.png">%s</a>' \ - "${reason}" \ - "${reason}" - elif [ "${reason% *}" != "${reason}" ]; then - printf '%s' \ - "${reason}" - else - printf '<font color="red">%s</font>' \ - "${reason}" - fi - if read -r operator; then - printf ' %s ' "${operator}" - fi - done - else - echo "${blocked_reason}" - fi - done | \ - tr '\n' ' ' - else - printf ' ' - fi - printf '\n' - done | \ - sort -k6n,6 | \ - while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do - if [ "${buildable}" -eq 0 ]; then - left='(' - right=')' - else - unset left - unset right - fi - printf '<tr>' - mod_rev=$( - modification_revision_link "${mod_rev}" "${repo}" "${pkg}" - ) - build_error=$( - echo "${build_error}" | \ - sed 's|,|, |g' - ) - printf '<td>%s</td>' \ - '<a href="graphs/'"${pkg}"'.png">'"${left}${pkg}${right}"'</a>' \ - "<p style=\"font-size:8px\">${rev}</p>" \ - "<p style=\"font-size:8px\">${mod_rev}</p>" \ - "${repo}" \ - '<a href="build-logs/error/'"${log_file}"'">'"${count}"'</a>' \ - "${dependent}" \ - "${build_error}" \ - "${reason}" - printf '</tr>\n' - done - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/broken-packages.html" - - rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason" - + sed ' + s|$|<br>| + ' + printf '%s\n' \ + '</body>' \ + '</html>' +} > \ + "${tmp_dir}/todos.html" + +if [ -s "${tmp_dir}/todos" ]; then + sed ' + :a + N + /\n$/!ba + s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 | + s/\n$// + s/\n/\\n/g + ' -i "${tmp_dir}/todos" + while read -r file line desc; do + printf '%s %s %s\n' \ + "$(printf '%s' "${file}" | base64 -w0)" \ + "$(printf '%s' "${line}" | base64 -w0)" \ + "$(printf '%s' "${desc}" | base64 -w0)" + done < \ + "${tmp_dir}/todos" | \ + sponge "${tmp_dir}/todos" + # update todos + # shellcheck disable=SC2016 + while read -r file line desc; do + printf 'UPDATE IGNORE `todos`' + printf ' SET `todos`.`line`=from_base64("%s")' \ + "${line}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`description`=from_base64("%s");\n' \ + "${desc}" + + printf 'UPDATE IGNORE `todos`' + printf ' SET `todos`.`description`=from_base64("%s")' \ + "${desc}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`line`=from_base64("%s");\n' \ + "${line}" + done < \ + "${tmp_dir}/todos" | \ + mysql_run_query + # insert unfound todos + # shellcheck disable=SC2016 { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>Todos in the build scripts</title>' \ - '</head>' \ - '<body>' - find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ - -exec grep -nHF '' '{}' \; | \ - awk ' - { print $0 } - /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} - ' | \ - sed -n ' - s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ - T - N - s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ - :a - N - s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ - ta - s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ - p - ' | \ + printf 'SHOW CREATE TABLE `todos`' | \ + mysql_run_query | \ sed ' - :a - N - /\n$/!ba - s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n| - ' | \ - sed ' - s|$|<br>| + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' - printf '%s\n' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/todos.html" - - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>Blacklisted packages</title>' \ - '<link rel="stylesheet" type="text/css" href="/static/style.css">' \ - '</head>' \ - '<body>' \ - '<table>' - printf '<tr>' - printf '<th>%s</th>' \ - 'package' \ - 'reason' - printf '</tr>\n' - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \ - tar -Ox | \ + printf ';\n' + printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' + while read -r file line desc; do + printf '(' + printf 'from_base64("%s"),' \ + "${file}" \ + "${line}" \ + "${desc}" | \ + sed 's/,$/),/' + done < \ + "${tmp_dir}/todos" | \ sed ' - s@FS#\([0-9]\+\)@<a href="https://bugs.archlinux.org/task/\1">\0</a>@ - s@FS32#\([0-9]\+\)@<a href="https://bugs.archlinux32.org/index.php?do=details\&task_id=\1">\0</a>@ - /.#/!s/$/#/ - s|\(.\)#|\1</td><td>| - /^\s*#/{ - s/^\s*#\s*// - s|\s*\(</td><td>\)|</font></s>\1| - s/^/<s><font color="#808080">/ - } - s|^|<tr><td>| - s|$|</td></tr>| + s/,$// ' - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/blacklist.html" - - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>log of ssh connections from build slaves</title>' \ - '</head>' \ - '<body>' \ - '<table>' - printf '<tr>' - printf '<th>%s</th>' \ - 'time' \ - 'build slave' \ - 'command' \ - 'arguments' - printf '</tr>\n' - if [ -r "${work_dir}/ssh-log" ]; then - tac "${work_dir}/ssh-log" | \ - while read -r date time slave command arguments; do - printf '<tr>' - printf '<td>%s</td>' \ - "${date} ${time}" \ - "${slave}" \ - "${command}" \ - "${arguments}" - printf '</tr>\n' - done - fi - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/ssh-log.html" + printf ';\n' + printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) ' + printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` ' + printf 'FROM `td` ' + printf 'WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');\n' + + printf 'DELETE FROM `todos` WHERE NOT EXISTS (' + printf 'SELECT * FROM `td`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');' + printf 'DROP TABLE `td`;\n' + printf 'DELETE FROM `todo_links` WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`' + printf ') OR NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' + printf ');\n' + } | \ + mysql_run_query + rm -f "${tmp_dir}/todos" +fi - find "${tmp_dir}" -maxdepth 1 -type f | \ - while read -r file; do - cat "${file}" > \ - "${webserver_directory}/${file##*/}" - done +{ + printf '%s\n' \ + '<html>' \ + '<head>' \ + '<title>log of ssh connections from build slaves</title>' \ + '</head>' \ + '<body>' \ + '<table>' + printf '<tr>' + printf '<th>%s</th>' \ + 'time' \ + 'build slave' \ + 'command' \ + 'arguments' + printf '</tr>\n' + if [ -r "${work_dir}/ssh-log" ]; then + tac "${work_dir}/ssh-log" | \ + while read -r date time slave command arguments; do + printf '<tr>' + printf '<td>%s</td>' \ + "${date} ${time}" \ + "${slave}" \ + "${command}" \ + "${arguments}" + printf '</tr>\n' + done + fi + printf '%s\n' \ + '</table>' \ + '</body>' \ + '</html>' +} > \ + "${tmp_dir}/ssh-log.html" -else - cat "${tmp_dir}/build-master-status.html" -fi +find "${tmp_dir}" -maxdepth 1 -type f | \ + while read -r file; do + cat "${file}" > \ + "${webserver_directory}/${file##*/}" + done |