diff options
| -rwxr-xr-x | bin/bootstrap-mysql | 2 | ||||
| -rwxr-xr-x | bin/build-packages | 109 | ||||
| -rwxr-xr-x | bin/change-git-remotes | 10 | ||||
| -rwxr-xr-x | bin/check-mirrors | 18 | ||||
| -rwxr-xr-x | bin/check-opcodes | 4 | ||||
| -rwxr-xr-x | bin/clean-cache | 2 | ||||
| -rwxr-xr-x | bin/clean-dependencies | 2 | ||||
| -rwxr-xr-x | bin/create-build-support-package | 2 | ||||
| -rwxr-xr-x | bin/db-update | 91 | ||||
| -rwxr-xr-x | bin/delete-packages | 5 | ||||
| -rwxr-xr-x | bin/get-assignment | 4 | ||||
| -rwxr-xr-x | bin/harvest-commit-times | 8 | ||||
| -rwxr-xr-x | bin/ii-answer | 66 | ||||
| -rwxr-xr-x | bin/ii-connect | 4 | ||||
| -rwxr-xr-x | bin/ii-watch | 27 | ||||
| -rwxr-xr-x | bin/interpret-mail | 2 | ||||
| -rwxr-xr-x | bin/local-build-package | 31 | ||||
| -rwxr-xr-x | bin/manage-slaves | 13 | ||||
| -rwxr-xr-x | bin/nit-picker | 122 | ||||
| -rwxr-xr-x | bin/ping-to-master | 2 | ||||
| -rwxr-xr-x | bin/return-assignment | 42 | ||||
| -rwxr-xr-x | bin/slave-build-connect | 6 | ||||
| -rwxr-xr-x | bin/systemd-email | 2 | ||||
| -rwxr-xr-x | bin/update-gpg-keys | 49 | ||||
| -rwxr-xr-x | conf/master.conf.example | 2 | ||||
| -rwxr-xr-x | lib/common-functions | 13 | ||||
| -rwxr-xr-x | lib/intentions-queue | 14 | ||||
| -rwxr-xr-x | lib/load-configuration | 23 | ||||
| -rwxr-xr-x | lib/mysql-functions | 27 | ||||
| -rw-r--r-- | misc/README.Indexes | 1 |
30 files changed, 480 insertions, 223 deletions
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 711f3d1..fbcbc47 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -695,7 +695,7 @@ fi printf 'DELIMITER ;\n' - if [ "x$1" != 'x-no-privileges' ]; then + if [ "$1" != '-no-privileges' ]; then printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ 'CREATE ROUTINE' 'buildmaster.*' \ 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ diff --git a/bin/build-packages b/bin/build-packages index 46c1ef1..de01478 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -78,6 +78,8 @@ usage() { >&2 echo ' compile from source tarbal from sources.archlinux.org if possible' >&2 echo ' :mirrored_source_by_hash:' >&2 echo ' download sources from sources.archlinux32.org by hash if possible' + >&2 echo ' :on_x86_64:' + >&2 echo ' build any package with x86_64 arch' >&2 echo ' :without_check:' >&2 echo ' run makepkg with "--no-check"' >&2 echo ' :without_systemd_nspawn:' @@ -249,7 +251,7 @@ while [ "${count}" -ne 0 ] && \ if [ -z "${forced_package}" ]; then package=$( # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -291,6 +293,9 @@ while [ "${count}" -ne 0 ] && \ package="${package% *}" if [ "${arch}" = 'any' ]; then arch="${my_arch}" + assignment_was_any=true + else + assignment_was_any=false fi if [ -n "${diff_source_dir}" ] && @@ -298,7 +303,7 @@ while [ "${count}" -ne 0 ] && \ >&2 echo 'The prefered package was not handed out.' >&2 echo 'Because -d was given, I will abort.' # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -309,9 +314,10 @@ while [ "${count}" -ne 0 ] && \ if [ "${git_revision##*-}" = 'HEAD' ]; then git_revision=$( repo_name="${git_revision%-*}" - eval repo_path='"${repo_paths__'"${repo_name}"'}"' - if [ -z "${repo_path}" ]; then - >&2 printf 'Unknown git repository "%s".\n' "${repo_name}" + eval l_repo_path='"${repo_paths__'"${repo_name}"'}"' + # shellcheck disable=SC2154 + if [ -z "${l_repo_path}" ]; then + >&2 printf 'Unknown git repository "%s".\n' "${l_repo_name}" exit 2 fi git -C "${repo_path}" rev-parse HEAD @@ -382,7 +388,7 @@ while [ "${count}" -ne 0 ] && \ ! apply_trunk_patch "${tmp_dir}" "${diff_source_dir}"; then # report local failure (probably a missing commit) to build-master # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -458,7 +464,7 @@ while [ "${count}" -ne 0 ] && \ >&2 echo 'The prefered package was not handed out.' >&2 echo 'Because straw :without_systemd_nspawn: is active, I will abort.' # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -484,6 +490,28 @@ while [ "${count}" -ne 0 ] && \ outerParameters="${innerParameters} -fcrs --asdeps --noconfirm --holdver" middleParameters='' innerParameters='' + elif echo "${straw}" | \ + grep -qF ':on_x86_64:'; then + if [ -z "${prefered_package}" ]; then + >&2 echo 'straw :on_x86_64: only allowed with -p' + exit 2 + fi + if [ "${prefered_package}" != "${package}" ]; then + >&2 echo 'The prefered package was not handed out.' + >&2 echo 'Because straw :on_x86_64: is active, I will abort.' + # shellcheck disable=SC2029 + ssh -o PasswordAuthentication=No \ + -i "${master_build_server_identity}" \ + -p "${master_build_server_port}" \ + "${master_build_server_user}@${master_build_server}" \ + 'return-assignment' 'ABORT' + exit 2 + fi + if ! ${assignment_was_any}; then + >&2 printf 'Can only build "any" packages with :on_x86_64:, but got a "%s" package.\n' "${arch}" + exit 2 + fi + build_command='staging-x86_64-build' else build_command='staging-'"${arch}"'-build' fi @@ -650,14 +678,15 @@ while [ "${count}" -ne 0 ] && \ t s/^See ['"'"'`]\([^'"'"'`]\+\)['"'"'`]\(\s.*\)\?$/\1/ t + s/^A full log can be found at\s\+\(\S\+\)\s*$/\1/ + t d ' "${log_file}" | \ while read -r extra_log_file; do - grep -HF '' "${build_dir}/${extra_log_file}" || true - if [ -n "${extra_log_file##*/*}" ]; then + if ! grep -HF '' "${build_dir}/${extra_log_file}"; then find "${build_dir}" \ - -xdev -mindepth 2 \ - -type f -name "${extra_log_file}" \ + -xdev \ + -type f -name "${extra_log_file##*/}" \ -exec grep -HF '' {} + \ || true fi @@ -677,8 +706,20 @@ while [ "${count}" -ne 0 ] && \ # machine, if need arises in the future). Note, that it is no # error, if the above build command already produced a # compressed package. - find . -maxdepth 1 -type f -name '*.pkg.tar' \ - -execdir zstd --rm -T0 --ultra -20 '{}' -o '{}.zst' \; + # Since --ultra -20 seems too much for our i486 build vms, we + # silently fall back to lower compression levels if the higher + # level fails. + + # beware: We must remove *.pkg.tar.zst after failed + # compression attempts, otherwise future compression attempts + # will refuse to overwrite them. + + # shellcheck disable=SC2086 + for options in '-T0 --ultra -20' '--ultra -20' $(seq -19 -3); do + find . -maxdepth 1 -type f -name '*.pkg.tar' \ + -not -execdir zstd --rm ${options} '{}' -o '{}.zst' \; \ + -exec rm -f '{}.zst' \; + done # package files, that still exist, threw an error upon compression errors=$( find . -maxdepth 1 -type f -name '*.pkg.tar' @@ -697,15 +738,15 @@ while [ "${count}" -ne 0 ] && \ # compressing successful echo 'post-build' > "${tmp_dir}/.ping-build-master" >&2 printf ' ok.\n' - if [ "${build_command}" = 'makepkg' ]; then - find . -maxdepth 1 -type f -name '*.pkg.tar.zst' \ - -exec sh -c 'namcap "$1" > "$1-namcap.log"' '_' '{}' \; - else - find . -maxdepth 1 -type f -name '*.pkg.tar-namcap.log' \ - | while read -r file; do - mv "${file}" "${file%-namcap.log}.zst-namcap.log" - done - fi + find . -maxdepth 1 -type f -name '*.pkg.tar.zst' \ + -exec sh -c ' + pkg="${1%.pkg.tar.zst}" + if [ -f "${pkg}.pkg.tar-namcap.log" ]; then + mv "${pkg}.pkg.tar-namcap.log" "${pkg}.pkg.tar.zst-namcap.log" + elif [ ! -f "${pkg}.pkg.tar.zst-namcap.log" ]; then + namcap "$1" > "${pkg}.pkg.tar.zst-namcap.log" + fi + ' '_' '{}' \; tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX") # remove unexpected packages if [ -n "${expected_packages}" ]; then @@ -716,11 +757,19 @@ while [ "${count}" -ne 0 ] && \ } | \ sort | \ uniq -u | \ - while read -r unexpected_package; do - >&2 printf 'removing unexpected build artifact "%s"\n' \ - "${unexpected_package}" - rm "${unexpected_package}"* - done + { + removed_something_unexpected=false + while read -r unexpected_package; do + >&2 printf 'removing unexpected build artifact "%s"\n' \ + "${unexpected_package}" + rm "${unexpected_package}"* + removed_something_unexpected=true + done + if "${removed_something_unexpected}"; then + >&2 printf 'I was only expecting:\n%s\n' \ + "${expected_packages}" + fi + } fi >&2 printf 'signing package(s)\n' find . -maxdepth 1 -type f -name '*.pkg.tar.zst' \ @@ -1001,7 +1050,7 @@ while [ "${count}" -ne 0 ] && \ while ${upload_to_build_master}; do err=0 # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -1050,7 +1099,7 @@ while [ "${count}" -ne 0 ] && \ if ${upload_to_build_master}; then printf '%s@%s\n' \ "$(whoami)" \ - "$(hostname)" >> \ + "$(uname -n)" >> \ "${log}" gzip "${log}" else @@ -1064,7 +1113,7 @@ while [ "${count}" -ne 0 ] && \ while true; do err=0 # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ diff --git a/bin/change-git-remotes b/bin/change-git-remotes index bc8e3f2..fc50b7a 100755 --- a/bin/change-git-remotes +++ b/bin/change-git-remotes @@ -3,7 +3,7 @@ # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" -if [ $# -eq 1 ] && [ "x$1" = 'x-q' ]; then +if [ $# -eq 1 ] && [ "$1" = '-q' ]; then quiet=true elif [ $# -ne 0 ]; then printf 'change-git-remotes [-q]\n' @@ -63,6 +63,14 @@ fi '' 2 3 4 ) ;; + 'community'|'packages') + new_urls=$( + printf '%s\n' \ + 'git://github.com/archlinux/svntogit-'"${git_name}" \ + 'https://github.com/archlinux/svntogit-'"${git_name}" \ + 'git://git.archlinux.org/svntogit/'"${git_name}"'.git' + ) + ;; *) new_urls="${url}" ;; diff --git a/bin/check-mirrors b/bin/check-mirrors index d106201..aec33a0 100755 --- a/bin/check-mirrors +++ b/bin/check-mirrors @@ -143,14 +143,16 @@ elif [ $# -eq 1 ]; then cnt=$( curl -s "${ip_flag}" "${url}archisos/" ) || has_isos=0 - for suffix in 'i686' 'dual'; do - # shellcheck disable=SC2059 - if ! printf '%s\n' "${cnt}" | \ - grep -qF "$(printf "$(date '+archlinux-%Y.%m.%%02d-'"${suffix}"'.iso\n')" $(seq 31))" -; then - has_isos=0 - break - fi - done +# doesn't really work: dual doesn't exist anymore and the other test sometimes fails, sometimes succeeds? +# let's assume the precense of the 'archisos' directory is enough +# for suffix in 'i686' 'dual'; do +# # shellcheck disable=SC2059 +# if ! printf '%s\n' "${cnt}" | \ +# grep -qF "$(printf "$(date '+archlinux-%Y.%m.%%02d-'"${suffix}"'.iso\n')" $(seq 31))" -; then +# has_isos=0 +# break +# fi +# done else has_isos=0 last_sync='0' diff --git a/bin/check-opcodes b/bin/check-opcodes index e9f69a2..2949819 100755 --- a/bin/check-opcodes +++ b/bin/check-opcodes @@ -90,7 +90,7 @@ shift $((OPTIND-1)) PACKAGE=$1 -if test "x$PACKAGE" = "x"; then +if test "$PACKAGE" = ""; then echo "ERROR: Filename of a package required as argument" >&2 usage exit 1 @@ -122,7 +122,7 @@ debug "Checking for architecture: $ARCH ($OPCODE_ARGS).." # shellcheck disable=SC2044 for absfile in $(find $tmp_dir \( -regextype grep -regex '.*\.so\(\.[0-9.]\+\)\?' -type f \) -o \( -executable -type f \) ); do file=$(basename $absfile) - relfile=${absfile#$tmp_dir} + relfile=${absfile#"${tmp_dir}"} debug "Checking file: $relfile" if ! readelf -a $absfile > $tmp_dir/$file.elf 2>/dev/null; then debug "readelf failed, ignoring file" diff --git a/bin/clean-cache b/bin/clean-cache index 8e6fb3c..21f1370 100755 --- a/bin/clean-cache +++ b/bin/clean-cache @@ -9,7 +9,7 @@ if [ $# -eq 0 ]; then dummynator='sudo' -elif [ $# -eq 1 ] && [ "x$1" = 'x-n' ]; then +elif [ $# -eq 1 ] && [ "$1" = '-n' ]; then dummynator='echo' else >&2 echo 'usage: clean-cache [-n]' diff --git a/bin/clean-dependencies b/bin/clean-dependencies index 82d49b3..63e9a9a 100755 --- a/bin/clean-dependencies +++ b/bin/clean-dependencies @@ -67,7 +67,7 @@ trap 'rm "${deps_file}"' EXIT printf 'INSERT IGNORE INTO `bps`(`bp`)' printf ' SELECT `deps`.`bp`' printf ' FROM `deps`;\n' - if [ $# -eq 1 ] && [ "x$1" = 'x-f' ]; then + if [ $# -eq 1 ] && [ "$1" = '-f' ]; then printf 'DELETE `dependencies`' else printf 'SELECT *' diff --git a/bin/create-build-support-package b/bin/create-build-support-package index 09374ec..43fb592 100755 --- a/bin/create-build-support-package +++ b/bin/create-build-support-package @@ -169,7 +169,7 @@ if [ -n "${source_package}" ]; then fi new_pkg="${pkg%-*-*-*}" - new_pkg="${new_pkg}-shim${pkg#${new_pkg}}" + new_pkg="${new_pkg}-shim${pkg#"${new_pkg}"}" failsafe_rsync \ "${master_mirror_rsync_directory}/pool/${pkg}" \ diff --git a/bin/db-update b/bin/db-update index 7c5a8cf..88c567d 100755 --- a/bin/db-update +++ b/bin/db-update @@ -27,6 +27,8 @@ usage() { >&2 echo ' move possible packages from staging to testing.' >&2 echo '' >&2 echo 'possible options:' + >&2 echo ' -e|--everything:' + >&2 echo ' Force movement of every package and move nothing else.' >&2 echo ' -f|--force $arch/$repo/$pkgname:' >&2 echo ' Force movement of given package and move nothing else.' >&2 echo ' -h|--help:' @@ -50,19 +52,24 @@ usage() { >&2 echo ' Move forward any package which replaces no package whose' >&2 echo ' dependencies are all available somewhere.' >&2 echo ' Note, that this _may_ move _less_ packages.' + >&2 echo ' -s|--stuck:' + >&2 echo ' Force movement of every package, that is stuck for too' + >&2 echo ' long, and move nothing else.' >&2 echo ' -w|--wait:' >&2 echo ' If necessary, wait for lock blocking.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( - getopt -o f:hino:pw \ - --long force \ + getopt -o ef:hino:psw \ + --long everything \ + --long force: \ --long help \ --long ignore-insanity \ --long no-action \ --long only: \ --long progressive \ + --long stuck \ --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage @@ -73,11 +80,16 @@ ignore_insanity=false no_action_flag='' progressive=false force_pkgs='' +force_every_pkg=false +force_stuck_pkgs=false only='' while true do case "$1" in + -e|--everything) + force_every_pkg=true + ;; -f|--force) shift force_pkgs=$( @@ -109,6 +121,9 @@ do >&2 echo 'Do not use this option, it is currently broken.' exit 42 ;; + -s|--stuck) + force_stuck_pkgs=true + ;; -w|--wait) block_flag='' ;; @@ -129,8 +144,17 @@ if [ $# -ne 0 ]; then usage fi -if ${progressive} && \ - [ -n "${force_pkgs}" ]; then +number_of_force_flags=0 +${progressive} \ +&& number_of_force_flags=$((number_of_force_flags+1)) +[ -n "${force_pkgs}" ] \ +&& number_of_force_flags=$((number_of_force_flags+1)) +${force_every_pkg} \ +&& number_of_force_flags=$((number_of_force_flags+1)) +${force_stuck_pkgs} \ +&& number_of_force_flags=$((number_of_force_flags+1)) + +if [ ${number_of_force_flags} -gt 1 ]; then >&2 echo 'db-update: conflicting arguments' usage fi @@ -157,7 +181,7 @@ if [ -n "${only}" ]; then printf 's@^!%s\\(/.\\+\\)\\?$@' \ "${arch}" for arch2 in i486 i686 pentium4; do - if [ "${arch}" == "${arch2}" ]; then + if [ "${arch}" = "${arch2}" ]; then continue fi printf '%s\\1\\n' \ @@ -295,7 +319,9 @@ mysql_query_update_replaced_bpir() { # are for the same architecture if we joined them via # repository_moves, because only repositories with the same # architectures should be listed there. - if [ -n "${force_pkgs}" ]; then + if [ -n "${force_pkgs}" ] || \ + ${force_every_pkg} || \ + ${force_stuck_pkgs}; then printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \ 'moveable' 'replaced' printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' @@ -306,31 +332,46 @@ mysql_query_update_replaced_bpir() { printf ' FROM `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_repositories printf ' AND `repositories`.`is_on_master_mirror`' + printf ' AND `repositories`.`stability`=%s' \ + "${source_stability_id}" + printf ' AND `repositories`.`architecture`=%s' \ + "${arch_id}" mysql_join_repositories_architectures mysql_join_binary_packages_in_repositories_binary_packages printf ' JOIN `binary_packages` AS `f_bp`' printf ' ON `binary_packages`.`build_assignment`=`f_bp`.`build_assignment`' mysql_join_binary_packages_binary_packages_in_repositories 'f_bp' 'f_bpir' printf ' AND `f_bpir`.`repository`=`repositories`.`id`' - printf ' WHERE CONCAT(' - printf '`architectures`.`name`,"/",' - printf '`repositories`.`name`,"/",' - printf '`f_bp`.`pkgname`' - printf ') IN (' - # shellcheck disable=SC2086 - printf 'from_base64("%s"),' \ - ${force_pkgs} | \ - sed 's/,$/);\n/' - printf 'DELETE `moveable_bpir`' - printf ' FROM `moveable_bpir`' - printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`' - mysql_join_binary_packages_in_repositories_binary_packages - mysql_join_binary_packages_in_repositories_repositories - printf ' WHERE `repositories`.`stability`!=%s' \ - "${source_stability_id}" - printf ' OR `repositories`.`architecture`!=%s;\n' \ - "${arch_id}" + if [ -n "${force_pkgs}" ]; then + printf ' WHERE CONCAT(' + printf '`architectures`.`name`,"/",' + printf '`repositories`.`name`,"/",' + printf '`f_bp`.`pkgname`' + printf ') IN (' + # shellcheck disable=SC2086 + printf 'from_base64("%s"),' \ + ${force_pkgs} | \ + sed 's/,$/);\n/' + elif ${force_stuck_pkgs}; then + printf ' WHERE `binary_packages_in_repositories`.`last_moved` < DATE_SUB(NOW(),INTERVAL ' + # shellcheck disable=SC2154 + case "${source_stability_id}" in + "${repository_stability_ids__testing}") + printf '%s.5' "$((max_testing_duration+2))" + ;; + "${repository_stability_ids__staging}") + printf '2.5' + ;; + *) + >&2 printf 'unknown source stability id: %s\n' "${source_stability_id}" + printf '1000000' + ;; + esac + printf ' day);\n' + else # ${force_every_pkg} + printf ';\n' + fi + printf 'UPDATE `moveable_bpir`' printf ' JOIN `binary_packages_in_repositories`' printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`' diff --git a/bin/delete-packages b/bin/delete-packages index a2e932c..0026c88 100755 --- a/bin/delete-packages +++ b/bin/delete-packages @@ -152,6 +152,7 @@ export TMPDIR="${tmp_dir}" { printf 'CREATE TEMPORARY TABLE `bpir_to_delete` (`id` BIGINT, UNIQUE KEY (`id`));\n' printf 'INSERT IGNORE INTO `bpir_to_delete`' + # shellcheck disable=SC2030 if [ -n "${force_packages}" ]; then printf ' SELECT' printf ' `binary_packages_in_repositories`.`id`' @@ -159,7 +160,8 @@ export TMPDIR="${tmp_dir}" mysql_join_binary_packages_in_repositories_binary_packages mysql_join_binary_packages_in_repositories_repositories mysql_join_repositories_architectures - printf ' WHERE CONCAT(' + printf ' WHERE `repositories`.`is_on_master_mirror`' + printf ' AND CONCAT(' printf '`architectures`.`name`,"/",' printf '`repositories`.`name`,"/",' printf '`binary_packages`.`pkgname`' @@ -342,6 +344,7 @@ export TMPDIR="${tmp_dir}" } ' +# shellcheck disable=SC2031 if [ -n "${build_support}" ] && \ [ -w "${build_support}" ] && \ [ ! -p "${build_support}" ]; then diff --git a/bin/get-assignment b/bin/get-assignment index f2930c4..6abfaef 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -553,7 +553,7 @@ next_building=$( printf '`build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query + mysql_run_query 'unimportant' ) if [ -n "${next_building}" ]; then hand_out_assignment "${next_building}" @@ -579,7 +579,7 @@ count_pending=$( "${arch}" printf ';\n' } | \ - mysql_run_query + mysql_run_query 'unimportant' ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/harvest-commit-times b/bin/harvest-commit-times index a978b41..7546aa4 100755 --- a/bin/harvest-commit-times +++ b/bin/harvest-commit-times @@ -76,9 +76,9 @@ trap 'rm "${tmp_file}"' EXIT if ! ${i_am_the_master}; then tmp_remote_file=$( - ssh buildmaster "mktemp 'tmp.harvest-commit-times.remotely.XXXXXXXXXX' --tmpdir" + ssh -o PasswordAuthentication=No buildmaster "mktemp 'tmp.harvest-commit-times.remotely.XXXXXXXXXX' --tmpdir" ) - trap 'rm "${tmp_file}"; ssh buildmaster rm "${tmp_remote_file}"' EXIT + trap 'rm "${tmp_file}"; ssh -o PasswordAuthentication=No buildmaster rm "${tmp_remote_file}"' EXIT fi while true; do @@ -127,7 +127,7 @@ while true; do if [ -s "${tmp_file}" ]; then if ! ${i_am_the_master}; then - ssh buildmaster 'cat > "'"${tmp_remote_file}"'"' < \ + ssh -o PasswordAuthentication=No buildmaster 'cat > "'"${tmp_remote_file}"'"' < \ "${tmp_file}" fi # shellcheck disable=SC2016 @@ -149,7 +149,7 @@ while true; do if ${i_am_the_master}; then mysql_run_query 'unimportant' else - ssh buildmaster 'mysql buildmaster -N --raw --batch' + ssh -o PasswordAuthentication=No buildmaster 'mysql buildmaster -N --raw --batch' fi else >&2 printf '\n' diff --git a/bin/ii-answer b/bin/ii-answer index ed333cb..16fbb78 100755 --- a/bin/ii-answer +++ b/bin/ii-answer @@ -7,25 +7,43 @@ # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" -# welcome devs (in #archlinux32 only) -welcome_user_regex='^\S\+ -!- \(abaumann\|buildmaster\|deep42thought\|girls\|nit-picker\|tyzoid\)(.* has joined \S\+$' -new_users=$( - sed -i ' - s/'"${welcome_user_regex}"'/\1/ - T - w /dev/stdout - d - ' "${irc_dir}/#archlinux32/out" | \ - sort -u -) -if [ -n "${new_users}" ]; then - printf '%s\n' "${new_users}" | \ - sed ' - s/^.*$/Hi \0!\n!rq \0/ - s/Hi buildmaster!\n// - ' | \ - irc_say '' 'copy' -fi +# welcome devs (in #archlinux32 and #archlinux32-devops only) +welcome_user_regex='^\S\+ -!- \(abaumann\|deep42thought\)(.* has joined \S\+$' +for channel in '#archlinux32:Hi \0!\n!rq \0' '#archlinux32-devops:Hi \0!'; do + new_users=$( + sed -i ' + s/'"${welcome_user_regex}"'/\1/ + T + w /dev/stdout + d + ' "${irc_dir}/${channel%%:*}/out" \ + | grep -vxF "$( + { + printf '%s SNIP\n' $(($(date +%s)-5*60)) + cat "${irc_dir}/out" + cat "${irc_dir}/${channel%%:*}/out" + } \ + | sort -k1nr,1 \ + | sed ' + /^\S\+ SNIP$/q + ' \ + | sed ' + s/^\S\+\s\+-!- \([^([:space:]]\+\)\((\S\+\)\? has quit.*$/\1/ + t + d + ' \ + | sort -u + )" \ + | sort -u + ) + if [ -n "${new_users}" ]; then + printf '%s\n' "${new_users}" | \ + sed ' + s/^.*$/'"${channel#*:}"'/ + ' | \ + irc_say "${channel%%:*}" 'copy' + fi +done find "${irc_dir}" \ -type f \ @@ -42,10 +60,10 @@ find "${irc_dir}" \ sloppy_salutation='' fi # answer "why don't you?" - regex='^\S\+ <\(\S\+\)> '"${prefix}"'!\?why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) ' + regex='^\S\+ <\(\S\+\)> '"${prefix}"'!\?why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) \([^?]*\)?\?$' if grep -q "${regex}" "${out_file}"; then sed -i ' - s/'"${regex}"'/\1 \2 / + s/'"${regex}"'/\1 \2 \3/ T w /dev/stdout d @@ -148,7 +166,7 @@ find "${irc_dir}" \ irc_say "${channel}" 'copy' fi # answer "sanity-check" - regex='^\S\+ <\S\+> .*\([Gg]et.*straight\|[Pp]ull.*yourself.*together\)' + regex='^\S\+ <\S\+> .*\([Gg]et.*straight\|[Pp]ull.*yourself.*together\|[Cc]heck.*again\)' if grep "${regex}" "${out_file}" | \ grep -q "${sloppy_salutation}"; then sed -i "/${regex}/d" "${out_file}" @@ -201,11 +219,11 @@ find "${irc_dir}" \ fi if [ -z "${channel%%#*}" ]; then # answer "Shut up!" - regex='^\S\+ <\S\+> \(.* \)\?[Ss][Hh][Uu][Tt] \?[Uu][Pp]' + regex='^\S\+ <\S\+> \(.* \)\?([Ss][Hh][Uu][Tt] \?[Uu][Pp]\|pss\+t\|[Bb]e \(.* \)\?quiet' if grep "${regex}" "${out_file}" | \ grep -q "${sloppy_salutation}"; then sed -i "/${regex}/d" "${out_file}" - printf 'Sorry, I will do.\n' | \ + printf 'Sorry, I will be quiet.\n' | \ irc_say "${channel}" 'copy' printf '%s\n' "$(( $(date '+%s') + 3600 diff --git a/bin/ii-connect b/bin/ii-connect index 652f94a..1e09d11 100755 --- a/bin/ii-connect +++ b/bin/ii-connect @@ -20,6 +20,6 @@ while ! grep -q 'You are \(now identified for\|already logged in as\) ' "${irc_d sleep 5 done -# join channel -echo '/j #archlinux32' | \ +# join channels +printf '/j #archlinux32%s\n' '' '-devops' | \ sponge "${irc_dir}/in" diff --git a/bin/ii-watch b/bin/ii-watch index e72bf6a..ae008c4 100755 --- a/bin/ii-watch +++ b/bin/ii-watch @@ -10,6 +10,7 @@ checksum=$( ) next_connection_check=0 +next_intentions_check=0 last_seen=0 while pgrep -x 'ii' >/dev/null && \ @@ -22,7 +23,7 @@ while pgrep -x 'ii' >/dev/null && \ $(date +%s) + 60*5 )) fi - echo '/NAMES #archlinux32' \ + echo '/NAMES #archlinux32-devops' \ | sponge "${irc_dir}/in" fi if [ -z "${said}" ]; then @@ -37,12 +38,34 @@ while pgrep -x 'ii' >/dev/null && \ said=$( "${base_dir}/bin/ii-answer" ) + if [ ${next_intentions_check} -lt "$(date +%s)" ]; then + # Are there old intentions waiting for execution? + oldest_intention=$( + find "${intentions_directory}" \ + -maxdepth 1 \ + -type f \ + -name 'intention.*' \ + -mmin +1200 \ + -printf '%Ts\n' \ + | sort -n \ + | tail -n1 + ) + if [ -n "${oldest_intention}" ]; then + printf 'There is something intented for %s minutes.\n' $(( + $(date +%s) - oldest_intention + )) \ + | irc_say + next_intentions_check=$(( + $(date +%s) + 3600 + )) + fi + fi # When was the buildmaster seen the last time? last_seen=$( { echo 0 sed ' - s/^\([0-9]\+\) = #archlinux32\( \S\+\)* buildmaster\( \S\+\)*$/\1/ + s/^\([0-9]\+\) = #archlinux32-devops\( \S\+\)* buildmaster\( \S\+\)*$/\1/ t d ' "${irc_dir}/out" diff --git a/bin/interpret-mail b/bin/interpret-mail index 1d6bf7f..2580c00 100755 --- a/bin/interpret-mail +++ b/bin/interpret-mail @@ -1,6 +1,6 @@ #!/bin/sh -# shellcheck disable=SC2119,SC2120 +# shellcheck disable=SC2119,SC2120,SC3043 # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" diff --git a/bin/local-build-package b/bin/local-build-package index fd4fdab..e0b02dc 100755 --- a/bin/local-build-package +++ b/bin/local-build-package @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # build one package to test if modifications are ok (before opening a pull # request in https://github.com/archlinux32/packages) @@ -31,8 +31,15 @@ usage() { >&2 echo ' Specify tmpdir instead of using a random one, usually with -N' >&2 echo ' -N|--nobuild:' >&2 echo ' Assume packages have already been built, just upload them.' - >&2 echo ' -s|--signkey:' + >&2 echo ' --signkey:' >&2 echo ' Use explicit sign key instead of the one from the configuration.' + >&2 echo ' -s|--straw $straw:' + >&2 echo ' Use this straw instead of the preconfigured ones. -- May be' + >&2 echo ' given multiple times to allow using multiple straws.' + >&2 echo '' + >&2 echo 'known straws (separated by and enclosed in ":", sets of straws separated by " "):' + >&2 echo ' :with_build_support:' + >&2 echo ' allow using [build-support]' [ -z "$1" ] && exit 1 || exit "$1" } @@ -47,6 +54,7 @@ eval set -- "$( --long host \ --long tmpdir \ --long signkey \ + --long straw \ --long help \ -n "$(basename "$0")" -- "$@" || \ echo usage @@ -61,6 +69,7 @@ bootstrap_host='' bootstrap_dir='' tmp_dir=$(mktemp -d "${work_dir}/tmp.XXXXXX") signkey="${package_key}" +declare -A straws while true do case "$1" in @@ -93,10 +102,14 @@ do shift tmp_dir="$1" ;; - -s|--signkey) + --signkey) shift signkey="$1" ;; + -s|--straw) + shift + straws["$1"]=1 + ;; -h|--help) usage 0 ;; @@ -162,8 +175,14 @@ if ! ${nobuild}; then if [ -z "${mod_git_revision}" ]; then mod_git_revision=$(git -C "${repo_paths__archlinux32}" rev-parse HEAD) fi - build_command="staging-${arch}-build" - parameters='' + + if [[ -z "${straws[:with_build_support:]}" ]]; then + build_command="staging-${arch}-build" + else + build_command='staging-with-build-support-'"${arch}"'-build' + fi + + parameters="-r ${archbuild_chroots}" if ${nocheck}; then parameters='-- -- --nocheck' fi @@ -212,7 +231,7 @@ if ${bootstrap}; then esac scp -P "${bootstrap_port}" -rC "${tmp_dir}/"*"-${arch}.pkg.tar.zst" "${tmp_dir}/"*"-${arch}.pkg.tar.zst.sig" "${bootstrap_host}:${bootstrap_dir}/${staging_repo}/." - ssh -p "${bootstrap_port}" "${bootstrap_host}" bash -l -c "'cd ${bootstrap_dir}/${staging_repo} && repo-add -n bootstrap-${staging_repo}.db.tar.gz *-${arch}.pkg.tar.zst'" + ssh -o PasswordAuthentication=No -p "${bootstrap_port}" "${bootstrap_host}" bash -l -c "'cd ${bootstrap_dir}/${staging_repo} && repo-add -n bootstrap-${staging_repo}.db.tar.gz *-${arch}.pkg.tar.zst'" fi # do not delete build reports, why might actually to want to have a look for things diff --git a/bin/manage-slaves b/bin/manage-slaves index cddda84..32883a3 100755 --- a/bin/manage-slaves +++ b/bin/manage-slaves @@ -9,6 +9,7 @@ usage() { >&2 echo 'manage-slaves $action [parameters]: manage the list of build slaves' >&2 echo '' >&2 echo 'possible actions:' + >&2 echo ' list' >&2 echo ' add $name $owner $ssh-key-fingerprint' >&2 echo ' disable $name' >&2 echo ' enable $name' @@ -33,6 +34,18 @@ if errors=$( fi case "$1" in + 'list') + # shellcheck disable=SC2016 + { + printf 'SELECT' + printf ' name, CASE WHEN access_allowed = 1 THEN "enabled" ELSE "disabled" END ' + printf ' name, CASE WHEN is_sane = 1 THEN "sane" ELSE "not sane" END ' + printf ' FROM build_slaves' + printf ';\n' + } | \ + mysql_run_query | \ + tr '\t' ' ' + ;; 'add') shift if [ $# -ne 3 ]; then diff --git a/bin/nit-picker b/bin/nit-picker index d356011..ed5b7da 100755 --- a/bin/nit-picker +++ b/bin/nit-picker @@ -10,34 +10,65 @@ # resilient against transient states of the buildmaster): # - check sanity of git # - check expiration of package and master keys +# - complain about packages residing too long in [staging]/[testing] +# - check if our packages are still available upstream # TODO: # - check for differences of dependencies between mysql and git # - check for differences of dependencies between mysql and packages # - check for installability of packages -# - check if our packages are still available upstream -# - complain about packages residing too long in [staging]/[testing] -if [ $# -ge 1 ] && [ "x$1" = 'x-n' ]; then - >&2 echo 'not joining irc' - irc=false - repair_dependencies=false - shift -elif [ $# -ge 1 ] && [ "x$1" = 'x-r' ]; then - >&2 echo 'repair all wrong dependencies (remove superfluid, add missing)' - irc=false - repair_dependencies=true +# shellcheck disable=SC2016 +print_usage() { + >&2 echo 'usage: nit-picker [-n|-r|-x] [$single_test]' + >&2 echo ' -n: Do not join irc.' + >&2 echo ' -r: Remove superfluid dependencies in the database.' + >&2 echo ' To avoid catastrophic breakage, this option requires' + >&2 echo ' $single_test being set.' + >&2 echo ' -x: Exit upon first found inconsistency.' + >&2 echo ' $single_test: Only execute the given test.' +} + +exit_fast=false +irc=true +repair_dependencies=false + +while [ $# -ge 1 ]; do + case "$1" in + '-n') + >&2 echo 'not joining irc' + irc=false + ;; + '-r') + >&2 echo 'repair all wrong dependencies (remove superfluid, add missing)' + repair_dependencies=true + ;; + '-x') + >&2 echo 'exit, when the first inconsistency is being found' + exit_fast=true + ;; + '-*') + >&2 printf 'unknown option "%s"\n' "$1" + print_usage + exit 1 + ;; + *) + break + ;; + esac shift +done + +if [ $# -eq 0 ]; then + single_run=false else - irc=true - repair_dependencies=false - # shellcheck disable=SC2016 - if [ $# -ne 0 ]; then - >&2 echo 'usage: nit-picker [-n|-r [$single_test]]' - >&2 echo ' -n: do not join irc' - >&2 echo ' -r: remove superfluid dependencies in the database' - >&2 echo ' $single_test: only execute the given test' - exit 1 - fi + single_run=true +fi + +if ${repair_dependencies} \ +&& ! ${single_run}; then + >&2 echo 'Option "-r" needs a test.' + print_usage + exit 1 fi clean_up() { @@ -55,7 +86,7 @@ if ${irc}; then rm -rf --one-file-system "${irc_dir}" - ii -s irc.freenode.net -n nit-picker -f nit-picker >/dev/null 2>&1 & + ii -s irc.libera.chat -n nit-picker -f nit-picker >/dev/null 2>&1 & ii_pid=$! clean_up() { @@ -78,10 +109,10 @@ if ${irc}; then done # join channel - echo '/j #archlinux32' | \ + echo '/j #archlinux32-devops' | \ sponge "${irc_dir}/in" - while [ ! -f "${irc_dir}/#archlinux32/out" ]; do + while [ ! -f "${irc_dir}/#archlinux32-devops/out" ]; do sleep 1 done @@ -100,6 +131,12 @@ local_irc_say() { else sed 's/^/irc: /' fi + if ${exit_fast}; then + exit 1 + fi + if ! ${single_run}; then + sleep 60 + fi } mysql_load_min_and_max_versions @@ -114,7 +151,9 @@ while pgrep -x ii >/dev/null \ else do_once_a_day_checks=false fi - if [ $# -eq 0 ]; then + if ${single_run}; then + printf '%s\n' "$*" + else # shellcheck disable=SC2016 { if ! "${repair_dependencies}"; then @@ -233,19 +272,18 @@ while pgrep -x ii >/dev/null \ mysql_run_query | \ tr '\t' ' ' | \ shuf - else - printf '%s\n' "$*" fi | \ while read -r action parameters; do if ${irc} && ! pgrep -x ii >/dev/null; then + >&2 echo 'ii is no longer running' break fi case "${action}" in 'commit') # check whether a given commit is present in the git repo git_repo="${parameters%% *}" - git_rev="${parameters#${git_repo} }" + git_rev="${parameters#"${git_repo}" }" git_head="${git_rev%% *}" - git_rev="${git_rev#${git_head} }" + git_rev="${git_rev#"${git_head}" }" # shellcheck disable=SC2016 eval "$( printf 'git_dir="${repo_paths__%s}"\n' \ @@ -373,9 +411,6 @@ while pgrep -x ii >/dev/null \ if ! ${irc}; then diff -u --color "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps" || true fi - if [ $# -eq 0 ]; then - sleep 60 - fi fi fi rm \ @@ -449,9 +484,6 @@ while pgrep -x ii >/dev/null \ if [ -n "${error_message}" ]; then printf '%s\n' "${error_message}" \ | local_irc_say - if [ $# -eq 0 ]; then - sleep 60 - fi fi rm \ "${tmp_dir}/${parameters}" \ @@ -507,9 +539,6 @@ while pgrep -x ii >/dev/null \ )" \ | awk '{print $1 ": your slave " $2 " builds " $3 "/" $4 " for more than a day, now (" ($5-1) " day(s) " $6 ")"}' \ | local_irc_say - if [ $# -eq 0 ]; then - sleep 60 - fi ;; 'keyring') if ! ${master_mirror_rsync_command} \ @@ -545,9 +574,6 @@ while pgrep -x ii >/dev/null \ "$(date -I -d@"${expiration}")" \ "${expiration_days}" \ | local_irc_say - if [ $# -eq 0 ]; then - sleep 60 - fi fi done done @@ -603,9 +629,6 @@ while pgrep -x ii >/dev/null \ s/^\S\+ /parts of \0are in different repositories: / ' \ | local_irc_say - if [ $# -eq 0 ]; then - sleep 60 - fi fi ;; 'upstream-availability') @@ -670,9 +693,6 @@ while pgrep -x ii >/dev/null \ printf 'Package %s is still in our repositories or on the build-list, but is not available upstream.\n' \ "${parameters}" \ | local_irc_say - if [ $# -eq 0 ]; then - sleep 60 - fi ;; 'repository-duration') case "${parameters#* }" in @@ -742,9 +762,6 @@ while pgrep -x ii >/dev/null \ fi } \ | local_irc_say - if [ $# -eq 0 ]; then - sleep 60 - fi ;; *) >&2 printf 'action "%s" is not yet implemented.\n' "${action}" @@ -756,7 +773,8 @@ while pgrep -x ii >/dev/null \ ;; esac done - if [ $# -ge 1 ] || "${repair_dependencies}"; then + if ${single_run}; then + >&2 echo 'only one run was requested' break fi sleep 120 @@ -863,6 +881,6 @@ done printf ';\n' fi } \ -| ifne ssh buildmaster 'mysql buildmaster +| ifne ssh -o PasswordAuthentication=No buildmaster 'mysql buildmaster rm -f "/tmp/add-those-dependencies" ' diff --git a/bin/ping-to-master b/bin/ping-to-master index b1539cc..0b72935 100755 --- a/bin/ping-to-master +++ b/bin/ping-to-master @@ -27,7 +27,7 @@ while kill -0 "${parent_pid}" && \ -name '*.build-log' \ -exec wc -l {} \; | \ sed 's, .*/, ,' | \ - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ diff --git a/bin/return-assignment b/bin/return-assignment index 6b64bf4..eca5af7 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -142,25 +142,25 @@ if [ "$6" = 'ERROR' ]; then exit 2 fi build_assignment_architecture="${infos##* }" - infos="${infos% ${build_assignment_architecture}}" + infos="${infos% "${build_assignment_architecture}"}" pkgrel="${infos##* }" - infos="${infos% ${pkgrel}}" + infos="${infos% "${pkgrel}"}" epoch="${infos##* }" - infos="${infos% ${epoch}}" + infos="${infos% "${epoch}"}" pkgver="${infos##* }" - infos="${infos% ${pkgver}}" + infos="${infos% "${pkgver}"}" pkgver=$( printf '%s' "${pkgver}" \ | base64 -d ) upstream_flag_date="${infos##* }" - infos="${infos% ${upstream_flag_date}}" + infos="${infos% "${upstream_flag_date}"}" upstream_flag_date=$( printf '%s' "${upstream_flag_date}" \ | base64 -d ) was_broken_before="${infos##* }" - build_assignment_id="${infos% ${was_broken_before}}" + build_assignment_id="${infos% "${was_broken_before}"}" # save sent build logs saved_build_logs=$( @@ -314,7 +314,10 @@ if [ "$6" = 'ERROR' ]; then ' | \ tr ' ' '\n' | \ sed ' - 1,2 d + /^Packages$/ { + N + d + } /^\s*$/d p s/^haskell-// @@ -373,6 +376,18 @@ if [ "$6" = 'ERROR' ]; then tr ' ' '\t' > \ "${tmp_dir}/broken-packages-with-version" +# TODO: We might want to prioritize broken dependencies, which are already +# on the build-list. + +# TODO: We might also want to prioritize the returned package if there +# were broken dependencies identified, but none of these is still on the +# build list (e.g. some race condition between this build and its +# dependencies occured). + +# TODO: Iff broken dependencies were rescheduled or prioritized, we +# should raise the priority of the returned package to one less than the +# maximum value (so it will be rebuilt after the broken dependencies). + # now we look if the broken packages have been rebuilt in the meantime haskell_rebuild_packages=$( # shellcheck disable=SC2016 @@ -517,10 +532,7 @@ if [ "$6" = 'ERROR' ]; then { printf '%s/%s ' \ "${build_assignment_architecture}" \ - "$1" \ - | sed ' - s/[+]/%2B/g - ' + "$1" if [ "${1%s}s" = "$1" ]; then printf 'are' else @@ -532,9 +544,6 @@ if [ "$6" = 'ERROR' ]; then # shellcheck disable=SC2154 printf ' broken (says %s)' \ "${slave}" - if [ "$1" = 'electron' ] || [ "$1" = 'electron2' ]; then - printf -- ' - as usual' - fi if [ -n "${rescheduled_packages}" ]; then printf -- ' - I rescheduled:' # shellcheck disable=SC2086 @@ -547,7 +556,10 @@ if [ "$6" = 'ERROR' ]; then fi printf ': https://archlinux32.org/buildmaster/build-log.php?a=%s&p=%s\n' \ "${build_assignment_architecture}" \ - "$1" + "$1" \ + | sed ' + s/[+]/%2B/g + ' } | \ irc_say fi diff --git a/bin/slave-build-connect b/bin/slave-build-connect index e40dc32..40a227c 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -5,9 +5,9 @@ # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" -if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ - [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xreturn-assignment' ] || \ - [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xping-from-slave' ]; then +if [ "${SSH_ORIGINAL_COMMAND%% *}" = 'get-assignment' ] || \ + [ "${SSH_ORIGINAL_COMMAND%% *}" = 'return-assignment' ] || \ + [ "${SSH_ORIGINAL_COMMAND%% *}" = 'ping-from-slave' ]; then # small check to prevent some shell-injections if printf '%s\n' "${SSH_ORIGINAL_COMMAND}" | \ diff --git a/bin/systemd-email b/bin/systemd-email index 8444758..4489c8d 100755 --- a/bin/systemd-email +++ b/bin/systemd-email @@ -10,7 +10,7 @@ fi printf 'To: archlinux32-logs@eckner.net\n' printf 'From: systemd <%s@%s>\n' \ "$(whoami)" \ - "$(hostname)" + "$(uname -n)" printf 'Subject: %s\n' \ "$1" printf 'Content-Transfer-Encoding: 8bit\n' diff --git a/bin/update-gpg-keys b/bin/update-gpg-keys new file mode 100755 index 0000000..fe6dfe3 --- /dev/null +++ b/bin/update-gpg-keys @@ -0,0 +1,49 @@ +#!/bin/sh + +# shellcheck source=../lib/load-configuration +. "${0%/*}/../lib/load-configuration" + +if [ $# -ne 0 ]; then + >&2 echo 'error: No arguments expected, but some were given.' + exit 1 +fi + +# shellcheck disable=SC2016 +{ + printf 'SELECT' + printf ' `gpg_keys`.`fingerprint`,' + printf 'replace(to_base64(`gpg_keys`.`public_key`),"\\n","")' + printf ' FROM `gpg_keys`' +} \ +| mysql_run_query \ +| while read -r fingerprint public_key; do + printf '%s\n' "${public_key}" \ + | base64 -d \ + | gpg --import \ + >/dev/null 2>&1 + gpg --quiet --locate-keys "${fingerprint}" \ + >/dev/null 2>&1 \ + || true + curl -Ss 'https://archlinux32.org/keys.php?k='"${fingerprint}" \ + | gpg --import --quiet \ + >/dev/null 2>&1 \ + || true + printf 'UPDATE `gpg_keys`' + printf ' SET `gpg_keys`.`public_key`=from_base64("%s")' \ + "$( + gpg -a --export "${fingerprint}" \ + | base64 -w0 + )" + printf ' WHERE `gpg_keys`.`fingerprint`="%s";\n' \ + "${fingerprint}" +done \ +| mysql_run_query + +# shellcheck disable=SC2016 +{ + printf 'SELECT `gpg_keys`.`fingerprint`' + printf ' FROM `gpg_keys`;\n' +} \ +| mysql_run_query \ +| xargs -r gpg -a --export \ +| curl -T - 'https://archlinux32.org/keys.php' diff --git a/conf/master.conf.example b/conf/master.conf.example index 4ec36f8..2234dfd 100755 --- a/conf/master.conf.example +++ b/conf/master.conf.example @@ -29,7 +29,7 @@ #intentions_directory="${work_dir}/intentions" # irc client configuration -#irc_dir='/home/master/irc/irc.freenode.net' +#irc_dir='/home/master/irc/irc.libera.chat' #irc_password='top_secret' # list of urls to trigger updates of the respective mirrors diff --git a/lib/common-functions b/lib/common-functions index 8b842ff..eb61e9e 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -2,7 +2,7 @@ # contains functions used by more than one script -# shellcheck disable=SC2039,SC2119,SC2120 +# shellcheck disable=SC2039,SC2119,SC2120,SC3043 if [ -z "${base_dir}" ]; then # just to make shellcheck happy @@ -277,8 +277,8 @@ remove_old_package_versions() { # wait between minimum and minimum+diff seconds (diff defaults to 30) wait_some_time() { - local minimum=$1 - local diff=$2 + local minimum="$1" + local diff="$2" local random if [ -z "${diff}" ]; then @@ -358,6 +358,7 @@ make_source_info() { p ' "${tmp_dir}/SRCINFO" )" + # shellcheck disable=SC2031 sed -i ' /^pkgname = /! b /= gtk-doc$/ b @@ -447,7 +448,7 @@ mangle_pkgbuild() { /^arch=[^#]*any/!{ /^arch=(/s/(/(i486 i686 pentium4 / } - s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ + s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9]\+\)\(\.[0-9.]*\)\?['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel:-\\3}"'"/ ' "${PKGBUILD}" # shellcheck disable=SC2016 @@ -778,7 +779,7 @@ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { } # irc_say $channel [copy] -# say content of stdin in irc channel $channel (default: #archlinux32) +# say content of stdin in irc channel $channel (default: #archlinux32-devops) # and print copy to stdout if 'copy' is given # shellcheck disable=SC2120 @@ -786,7 +787,7 @@ irc_say() { local channel local content if [ -z "$1" ]; then - channel='#archlinux32' + channel='#archlinux32-devops' else channel="$1" fi diff --git a/lib/intentions-queue b/lib/intentions-queue index c92a4d1..802d1ca 100755 --- a/lib/intentions-queue +++ b/lib/intentions-queue @@ -2,7 +2,7 @@ # contains functions related to the intentions-queue -# shellcheck disable=SC2039 +# shellcheck disable=SC2039,SC3043 if [ -z "${base_dir}" ]; then # just to make shellcheck happy @@ -15,7 +15,7 @@ fi # $package_database_lock_file externally intent_something() { local next_number - if [ "x$1" = 'x-n' ]; then + if [ "$1" = '-n' ]; then cat return fi @@ -75,7 +75,7 @@ execute_intention() { # with -n: return immediately # without -n: wait until nothing more to do intentions_left() { - if [ ! "x$1" = 'x-n' ]; then + if [ ! "$1" = '-n' ]; then while intentions_left -n; do sleep 1 done @@ -85,13 +85,7 @@ intentions_left() { -type f \ -name 'intention.*' \ -printf '%f\n' \ - | sed ' - s/^intention\.// - t - d - ' \ - | sort -n \ - | grep -qxm1 '[0-9]\+' + | grep -qxm1 'intention\.[0-9]\+' } # execute_all_intentions diff --git a/lib/load-configuration b/lib/load-configuration index b12b0cd..4c825f1 100755 --- a/lib/load-configuration +++ b/lib/load-configuration @@ -4,14 +4,10 @@ # shellcheck disable=SC2034 -# TODO: If there are intentions left which noone executes, we should try -# to reexecute them or at least warn the caller. Else, we might wait -# indefinitely. - set -e export LANG=C -if [ "$(hostname)" = 'buildmaster.archlinux32.org' ]; then +if [ "$(uname -n)" = 'buildmaster.archlinux32.org' ]; then i_am_the_master=true # abort early if mysqld is not running or kill switch active @@ -42,7 +38,7 @@ else fi # dirty hack to get this stuff debugable from a bash -if [ "x${0##*/}" = "x-bash" ] || [ "x${0##*/}" = "xbash" ] || [ "x${0##*/}" = "xdash" ]; then +if [ "${0##*/}" = "-bash" ] || [ "${0##*/}" = "bash" ] || [ "${0##*/}" = "dash" ]; then set +e if [ -z "${base_dir}" ]; then base_dir=$(pwd) @@ -118,7 +114,7 @@ webserver_directory='/srv/http' build_log_directory='/srv/http/build-logs' # irc client configuration -irc_dir='/home/master/irc/irc.freenode.net' +irc_dir='/home/master/irc/irc.libera.chat' irc_password='top_secret' # commands to run to update the package mirror used for installing archlinux32 packages, @@ -139,6 +135,12 @@ max_testing_duration="14" # set default location of archbuild chroots archbuild_chroots='/var/lib/archbuild' +# load static values from the database +if ${i_am_the_master} \ +|| [ "${0##*/}" = 'nit-picker' ]; then + mysql_retrieve_static_information +fi + # possibly pull in custom modifications if [ -r "${base_dir}/conf/common.conf" ]; then @@ -158,11 +160,6 @@ if ! ${i_am_the_master} && \ . "${base_dir}/conf/slave.conf" fi -# load static values from the database -if ${i_am_the_master}; then - mysql_retrieve_static_information -fi - # check / set up environment if [ -z "${build_list_lock_file}" ]; then @@ -216,7 +213,7 @@ for repo in ${repo_names} 'releng'; do repo_source='git@git.archlinux32.org:archlinux32/releng.git' ;; *) - repo_source="git://git.archlinux.org/svntogit/${repo}.git" + repo_source="https://github.com/archlinux/svntogit-${repo}.git" ;; esac git clone --mirror "${repo_source}" "${repo_path}" diff --git a/lib/mysql-functions b/lib/mysql-functions index 0c633c8..af2c3ba 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -2,7 +2,7 @@ # contains functions used to access mysql db -# shellcheck disable=SC2016,SC2039,SC2119,SC2120 +# shellcheck disable=SC2016,SC2039,SC2119,SC2120,SC3043 if [ -z "${base_dir}" ]; then # just to make shellcheck happy @@ -35,7 +35,7 @@ mysql_run_query() { local number file_name_extra='' - if [ "x$1" = 'xunimportant' ]; then + if [ "$1" = 'unimportant' ]; then shift file_name_extra='unimportant_' elif [ -s "${work_dir}/build-master-sanity" ]; then @@ -47,7 +47,7 @@ mysql_run_query() { query_stdout="${query_stdin%.stdin}.stdout" query_stderr="${query_stdin%.stdin}.stderr" cat > "${query_stdin}" - for _ in {1..10}; do + for _ in $(seq 10); do if [ -f "${query_stdout}" ]; then wait_some_time 10 10 fi @@ -78,7 +78,7 @@ mysql_run_query() { # a present query_file means there was an error if [ -f "${query_stdin}" ]; then >&2 printf 'I (%s) could not complete a mysql query:\n' \ - "$(hostname)" + "$(uname -n)" >&2 sed 's/^/mysql< /' "${query_stdin}" >&2 sed 's/^/mysql>2 /' "${query_stderr}" files="${query_stdin} ${query_stdout} ${query_stderr}$( @@ -221,7 +221,8 @@ mysql_generate_package_metadata() { mod_git_revision="$4" repository="$5" - if [[ "${current_repository_id}" = *[!0-9]* ]]; then + if printf '%s\n' "${current_repository_id}" \ + | grep -q '[^0-9]'; then >&2 printf 'mysql_generate_package_metadata(): invalid current_repository_id="%s".\n' \ "${current_repository_id}" exit 2 @@ -898,7 +899,7 @@ mysql_cleanup() { local operator if [ "$#" = '0' ]; then operator='DELETE' - elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then + elif [ "$#" = '1' ] && [ "$1" = 'dry' ]; then operator='SELECT COUNT(1)' else >&2 echo 'Unknown parameter' @@ -1440,6 +1441,7 @@ mysql_join_dependencies_install_target_providers_with_versions() { # - ids of compressions -> $compression_ids__$compression # - $package_compression_suffix_regex mysql_retrieve_static_information() { + local found_something eval "$( { printf 'SELECT CONCAT(' @@ -1513,9 +1515,16 @@ mysql_retrieve_static_information() { printf ') SEPARATOR "\\\\|"),' printf '"\\""' printf ') FROM `compressions`;\n' - } | \ - mysql_run_query + } \ + | sed ' + s@^\(SELECT \)\(.*\)$@SELECT "found_something=false;"; \1"found_something=true; ",\2 SELECT "if ! ${found_something}; then >\&2 echo \\"Some variable was not found in the database.\\"; exit 1; fi;";@ + ' \ + | mysql_run_query )" + if ! ${found_something}; then + >&2 echo 'No variables were found in the database.' + exit 1 + fi } # mysql_query_and_delete_unneeded_binary_packages @@ -1550,7 +1559,7 @@ mysql_sort_versions() { ( # new shell is intentional tmp_file=$(mktemp 'tmp.mysql_sort_versions.XXXXXXXXXX' --tmpdir) trap 'rm "${tmp_file}"' EXIT - if [ "x$1" != 'x-f' ] && \ + if [ "$1" != '-f' ] && \ { printf 'SELECT' printf ' COUNT(1)' diff --git a/misc/README.Indexes b/misc/README.Indexes new file mode 100644 index 0000000..ddf1267 --- /dev/null +++ b/misc/README.Indexes @@ -0,0 +1 @@ +create index mirror_statuses_last_sync on mirror_statuses(last_sync ASC); |