#!/bin/sh # report back on a build assignment # either on success via: # "$0 $package $revision $mod_revision $repository" and tar'ed packages and logs # (= a tar of package(s), signature(s) and log(s)) on stdin # or on failure via: # "$0 $package $revision $mod_revision $repository ERROR" and tar'ed logs # exit codes: # 0: ok # 1: another instance was already running # 2: outdated package # 3: signature error # 4: package error (e.g. wrong packages sent) # TODO: sign database # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit 1 fi # Create a lock file and a trap. exec 9> "${build_list_lock_file}" if ! flock -n 9; then >&2 echo 'come back (shortly) later - I cannot lock build list.' exit 1 fi exec 8> "${sanity_check_lock_file}" if ! flock -s -n 8; then >&2 echo 'come back (shortly) later - sanity-check running.' exit 1 fi clean_up_lock_file() { rm -f "${build_list_lock_file}" } trap clean_up_lock_file EXIT if [ "$5" = 'ERROR' ]; then # the build failed on the build slave # shellcheck disable=SC2016 infos=$( { printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`' mysql_join_build_slaves_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories mysql_join_build_assignments_binary_packages mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( # shellcheck disable=SC2154 printf '%s' "${slave}" | \ base64 -w0 )" printf ' AND `package_sources`.`%s`=from_base64("%s")' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ mysql_run_query | \ tr '\t' ' ' ) if [ -z "${infos}" ]; then >&2 echo 'You do not build this package (anymore) - move on.' exit 2 fi was_broken_before="${infos##* }" # save sent build logs saved_build_logs=$( tar -vx \ -C "${build_log_directory}/error" \ --wildcards \ --no-wildcards-match-slash \ --transform="s|^|$1.$2.$3.$4.|" \ '*.build-log.gz' ) # shellcheck disable=SC2154 echo "${slave}" >> \ "${work_dir}/package-states/$1.$2.$3.$4.broken" # shellcheck disable=SC2016 { if [ -n "${saved_build_logs}" ]; then printf 'CREATE TEMPORARY TABLE `failures` (' printf '`%s` %s,' \ 'date' 'DATETIME' \ 'reason' 'SMALLINT' \ 'log_file' 'VARCHAR(512)' | \ sed 's/,$//' printf ');\n' printf 'INSERT INTO `failures` (`date`,`reason`,`log_file`) VALUES' fail_reason_identifiers=$( { printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ mysql_run_query ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ while read -r reason_id identifier; do if zgrep -q "^$( printf '%s' "${identifier}" | \ base64 -d )\$" \ "${build_log_directory}/error/$1.$2.$3.$4.${saved_build_log}"; then printf ' (from_base64("%s"),%s,from_base64("%s")),' \ "$( printf '%s' "${saved_build_log}" | \ sed 's|\.build-log\.gz$||;s|^.*\.||' | \ base64 -w0 )" \ "${reason_id}" \ "$( printf '%s' "$1.$2.$3.$4.${saved_build_log}" | \ base64 -w0 )" break fi done done | \ sed 's/,$//' printf ';\n' printf 'INSERT INTO `failed_builds` (`build_slave`,`build_assignment`,`date`,`reason`,`log_file`)' printf ' SELECT ' printf '`build_slaves`.`%s`,' \ 'id' 'currently_building' printf '`failures`.`%s`,' \ 'date' 'reason' 'log_file' | \ sed 's/,$//' printf ' FROM `build_slaves` JOIN `failures`' printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" printf 'DROP TABLE `failures`;\n' fi printf 'UPDATE `build_assignments`' mysql_join_build_assignments_build_slaves printf ' SET `build_assignments`.`is_broken`=1, `build_slaves`.`currently_building`=NULL' printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" } | \ mysql_run_query # shellcheck disable=SC2154 sed -i ' /^'"$(str_to_regex "${slave}")"'$/d ' "${work_dir}/package-states/$1.$2.$3.$4.locked" if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then rm "${work_dir}/package-states/$1.$2.$3.$4.locked" # move that build order to the end of the build-list sed -i ' /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { $ b d } $ a '"$1 $2 $3 $4" \ "${work_dir}/build-list" fi # release lock on build-list - otherwise seed-build-list won't run flock -u 9 if ! ${was_broken_before}; then haskell_rebuild_packages=$( find "${build_log_directory}/error" -type f \ -name "$1.$2.$3.$4.*.build-log.gz" \ -exec zgrep -qFx ' The following packages are broken because other packages they depend on are missing. These broken packages must be rebuilt before they can be used.' {} \; \ -exec zcat {} \; | \ sed -n ' s/^installed package \(.*\) is broken due to missing package .*$/\1/ T p ' | \ tr ' ' '\n' | \ sed ' s/^/-p ^\\(haskell-\\)\\?/ s/-[0-9.]\+$/\$/ ' | \ sort -u ) rescheduled_packages=$( if [ -n "${haskell_rebuild_packages}" ]; then # shellcheck disable=SC2086 "${base_dir}/bin/seed-build-list" ${haskell_rebuild_packages} | \ sed 's/ .*$//' fi ) { printf '%s is broken (says %s).' \ "$1" \ "${slave}" if [ -n "${rescheduled_packages}" ]; then printf -- ' - I rescheduled:' # shellcheck disable=SC2086 printf ' %s,' ${rescheduled_packages} | \ sed 's/,$/./' fi printf '\n' } | \ irc_say fi exit 0 fi # the build was successful on the build slave # so we also need a lock on the package database exec 7> "${package_database_lock_file}" if ! flock -n 7; then >&2 echo 'come back (shortly) later - I cannot lock package database.' exit 1 fi clean_up_lock_file() { rm -f "${build_list_lock_file}" rm -f "${package_database_lock_file}" } # shellcheck disable=SC2016 if ! { printf 'SELECT count(*)' printf ' FROM `binary_packages`' mysql_join_binary_packages_build_assignments mysql_join_build_assignments_build_slaves mysql_join_binary_packages_repositories mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_slaves`.`name`=from_base64("%s")' \ "$(printf '%s' "${slave}" | base64 -w0)" printf ' AND `package_sources`.`%s`=from_base64("%s")' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ mysql_run_query | \ grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi clean_up_tmp_dir() { cd "${base_dir}" rm -rf --one-file-system "${tmp_dir}" clean_up_lock_file } tmp_dir=$(mktemp -d "${work_dir}/tmp.return-assignment.XXXXXXXXXX") cd "${tmp_dir}" trap clean_up_tmp_dir EXIT # extract package(s) tar -x \ --wildcards \ --no-wildcards-match-slash \ '*.pkg.tar.xz' \ '*.pkg.tar.xz.sig' \ '*.pkg.tar.xz-namcap.log.gz' \ '*.pkg.tar.xz.so.needs.gz' \ '*.pkg.tar.xz.so.provides.gz' # check if all packages come with: # - a package file # - a signature # - a namcap log # - a list of needed libraries # - a list of provided libraries missing_files=$( find . -maxdepth 1 \( \ \( \ -name '*.pkg.tar.xz' \ -printf '%f package\n' \ \) -o \ \( \ -name '*.pkg.tar.xz.sig' \ -printf '%f signature\n' \ \) -o \ \( \ -name '*.pkg.tar.xz-namcap.log.gz' \ -printf '%f namcap\n' \ \) -o \ \( \ -name '*.pkg.tar.xz.so.needs.gz' \ -printf '%f needed-libraries\n' \ \) -o \ \( \ -name '*.pkg.tar.xz.so.provides.gz' \ -printf '%f provided-libraries\n' \ \) \ \) | \ sed ' s/\(\.pkg\.tar\.xz\)\(\.sig\|-namcap\.log\.gz\|\.so\.\(provides\|needs\)\.gz\) /\1 / ' | \ sort -k1,1 -k2,2 | \ sed ' :a $!N s/^\(\(\S\+\) [^\n]\+\)\n\2 /\1 / ta P D ' | \ sed -n ' s/$/ / / package /!{ h s/^\(\S\+\) .*$/Package "\1" is missing./ p g } / signature /!{ h s/^\(\S\+\) .*$/Signature of "\1" is missing./ p g } / namcap /!{ h s/^\(\S\+\) .*$/Namcap log of "\1" is missing./ p g } / needed-libraries /!{ h s/^\(\S\+\) .*$/List of libraries needed by "\1" is missing./ p g } / provided-libraries /!{ h s/^\(\S\+\) .*$/List of libraries provided by "\1" is missing./ p g } ' ) if [ -n "${missing_files}" ]; then >&2 echo 'The following packages lack a signature, namcap log, package file or list of needed/provided libraries:' >&2 echo "${missing_files}" exit 3 fi # check if the signatures are valid signatures=$( find . -maxdepth 1 -name '*.pkg.tar.xz' \ -printf 'package file %f\n' \ -exec gpg --batch --status-fd 1 -q --homedir /etc/pacman.d/gnupg --verify '{}.sig' '{}' \; 2> /dev/null ) if [ -z "$( echo "${signatures}" | \ cut -d' ' -f2 | \ grep -x 'file\|TRUST_FULLY' | \ sort | \ uniq -c | \ awk '{print $1}' | \ uniq -d )" ]; then >&2 echo 'Signature(s) is/are not fully trusted:' >&2 echo "${signatures}" exit 3 fi # check if the sent packages are the expected ones packages=$( find . -maxdepth 1 -name '*.pkg.tar.xz' -printf '%f\n' ) package_errors=$( { # shellcheck disable=SC2086 printf '%s\n' ${packages} | \ sed ' s|^|was_built: | ' # shellcheck disable=SC2016 { printf 'SELECT CONCAT(' printf '"expected: ",' mysql_package_name_query printf ')' printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures mysql_join_binary_packages_build_slaves mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ mysql_run_query } | \ sort -k2 | \ uniq -u -f1 ) if [ -n "${package_errors}" ]; then >&2 echo 'The following packages should have been built but are missing or vice versa:' >&2 echo "${package_errors}" exit 4 fi # move namcap.logs find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_log_directory}/success/" \; # insert provided libraries into database # shellcheck disable=SC2016 { for lib_link in 'pl:provides' 'nl:needs'; do printf 'CREATE TEMPORARY TABLE `%s` (`pkgfile` VARCHAR(64), `lib` VARCHAR(128));\n' \ "${lib_link%:*}" find . -maxdepth 1 -name '*.pkg.tar.xz.so.'"${lib_link#*:}"'.gz' -execdir zgrep -HF '' '{}' \; | \ sed -n ' s,^\./\(.\+\.pkg\.tar\.xz\)\.so\.'"${lib_link#*:}"'\.gz:\([^:]\+\)$,\1\n\2, T p ' | \ base64_encode_each | \ sed ' N s/^\(\S\+\)\n\(\S\+\)$/(from_base64("\1"),from_base64("\2")),/ $s/,$/;/ 2 s/^/INSERT INTO `'"${lib_link%:*}"'` (`pkgfile`,`lib`) VALUES / ' printf 'INSERT IGNORE INTO `install_targets` (`name`)' printf ' SELECT DISTINCT `%s`.`lib` FROM `%s`;\n' \ "${lib_link%:*}" "${lib_link%:*}" if [ "${lib_link%:*}" = 'pl' ]; then printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`)' else printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`)' fi printf ' SELECT `binary_packages`.`id`,`install_targets`.`id`' if [ "${lib_link%:*}" = 'nl' ]; then printf ',`dependency_types`.`id`' fi printf ' FROM `install_targets`' if [ "${lib_link%:*}" = 'nl' ]; then printf ' JOIN `dependency_types` ON `dependency_types`.`name`="link"' fi printf ' JOIN `%s` ON `%s`.`lib`=`install_targets`.`name`' \ "${lib_link%:*}" "${lib_link%:*}" printf ' JOIN `binary_packages`' mysql_join_binary_packages_architectures mysql_join_binary_packages_build_slaves mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" printf ' AND `repositories`.`name`="build-list"' printf ' AND ' mysql_package_name_query printf '=`%s`.`pkgfile`' \ "${lib_link%:*}" printf ';\n' done } | \ mysql_run_query # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/${destination}/${destination}.db."* \ "${master_mirror_rsync_directory}/i686/${destination}/${destination}.files."* \ . # shellcheck disable=SC2086 repo-add "${destination}.db.tar.gz" ${packages} # repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages} ${master_mirror_rsync_command} \ "${destination}.db."* \ "${destination}.files."* \ ./*".pkg.tar.xz" \ ./*".pkg.tar.xz.sig" \ "${master_mirror_rsync_directory}/i686/${destination}/" trigger_mirror_refreshs for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done # remove old state files (these should be only "done" markers, but # actually we don't care what it is) - as long as it's not "testing" or "tested" find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ -not -name '*.testing' \ -not -name '*.tested' \ -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \ -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \ -exec rm '{}' \; # remove package from build list sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" # remove package lock file if ! [ "${destination}" = 'build-support' ]; then # shellcheck disable=SC2086 printf '%s\n' ${packages} > \ "${work_dir}/package-states/$1.$2.$3.$4.done" fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' mysql_join_build_assignments_build_slaves mysql_join_build_assignments_binary_packages mysql_join_binary_packages_repositories '' 'old_repo' printf ' SET' printf ' `build_assignments`.`is_broken`=0,' printf ' `build_assignments`.`priority`=0,' printf ' `binary_packages`.`repository`=(SELECT `new_repo`.`id` FROM `repositories` AS `new_repo` WHERE `new_repo`.`name`=from_base64("%s")),' \ "$( printf '%s' "${destination}" | \ base64 -w0 )" printf ' `binary_packages`.`has_issues`=0,' printf ' `binary_packages`.`is_tested`=0' printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" printf ' AND `old_repo`.`name`="build-list";\n' printf 'UPDATE `build_slaves` AS `to_update`' printf ' JOIN `build_slaves` AS `current_slave`' printf ' ON `to_update`.`currently_building`=`current_slave`.`currently_building`' printf ' SET `to_update`.`currently_building`=NULL' printf ' WHERE `current_slave`.`name`=from_base64("%s");\n' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n' printf 'INSERT INTO `loops_to_delete`' printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`' mysql_join_build_dependency_loops_binary_packages mysql_join_binary_packages_repositories printf ' WHERE NOT `repositories`.`name` = "build-list";\n' printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS (' printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`' printf ');\n' printf 'DROP TABLE `loops_to_delete`;\n' } | \ mysql_run_query rm -f \ "${work_dir}/package-states/$1.$2.$3.$4.locked" \ "${work_dir}/package-states/$1.$2.$3.$4.broken"