#!/bin/sh # receive one package to be built from the build-list whose dependencies # are already satisfied or which breaks a dependency cycle # exit code shows state of success: # 0: ok, I gave you an assignment # 1: come back (shortly) later - I was running already # 2: come back later - there are still packages to be built, # but currently none has all its dependencies ready # 3: come back after the next run of get-package-updates - currently # there are no pending packages # 4: come back, when you've done your work - you hit the limit on # maximum allowed parallel jobs per ip # TODO: # if all loops are locked, broken packages are not handed out as long # as the script believes there is a looped package before the broken # one on the build-list - clean up that mess! # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" mkdir -p "${work_dir}/package-states" hand_out_assignment() { # locked and blocked packages won't be handed out if package_locked_or_blocked "$1" "$2" "$3" "$4"; then return 0 fi # we don't care anymore if an older version of this package was # "locked" or "broken" (we keep only marker for older "done" packages) find "${work_dir}/package-states" -maxdepth 1 | \ grep "/$(str_to_regex "${1}")\(\.[^.]\+\)\{3\}\.\(locked\|broken\)\$" | \ grep -v "/$(str_to_regex "$1.$2.$3.$4.")[^.]\+\$" | \ xargs -rn1 rm -f echo "$1 $2 $3 $4" # shellcheck disable=SC2154 echo "${slave}" > "${work_dir}/package-states/$1.$2.$3.$4.locked" # lock every loop this package breaks grep -lxF "${1}" "${work_dir}/build-list.loops/"loop_* | \ tee -a "${work_dir}/package-states/$1.$2.$3.$4.locked" | \ sed 's|$|.locked|' | \ xargs -rn1 touch exit 0 } if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit 1 fi # Create a lock file and a trap. exec 9> "${build_list_lock_file}" if ! flock -n 9; then >&2 echo 'come back (shortly) later - I cannot lock build list.' exit 1 fi clean_up() { rm -f "${build_list_lock_file}" } trap clean_up EXIT # Check if there are any pending packages at all and if the requester # has already hit its max_parallel_build_per_client limit. num_jobs=0 pending_packages=false while read -r package git_revision mod_git_revision repository; do if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \ [ "${package}" = 'break_loops' ]; then continue fi generate_package_metadata "${package}.${git_revision}.${mod_git_revision}.${repository}" if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then continue fi if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then if [ "${slave}" = "$(head -n1 "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked")" ]; then num_jobs=$((num_jobs+1)); fi else pending_packages=true fi done < "${work_dir}/build-list" if ! ${pending_packages}; then >&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages' exit 3 fi if [ ${num_jobs} -ge ${max_parallel_build_per_client} ]; then >&2 echo "come back, when you've done your work - you hit the limit on maximum allowed parallel jobs per ip" exit 4 fi # Find first package of build-list whose dependencies are all met for hand_out_broken in false true; do # shellcheck disable=SC2094 while read -r package git_revision mod_git_revision repository; do if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \ [ "${package}" = 'break_loops' ]; then if ${hand_out_broken}; then first=true while read -r s; do if [ "${s}" = 'break_loops' ] && \ ${first}; then first=false continue fi printf '%s\n' "${s}" done < \ "${work_dir}/build-list" | \ sponge "${work_dir}/build-list" insert_break_loops_orders "${work_dir}/build-list" break else continue fi fi if package_locked_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then continue fi if ! ${hand_out_broken} && \ [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then continue fi if [ -n "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then continue fi hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" done < \ "${work_dir}/build-list" done # Find package (of all packages which are not locked and have been broken the least times) # which breaks the most unlocked loops locked_packages=$( find "${work_dir}/package-states/" -maxdepth 1 -printf '%f\n' | \ grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^\.\]\+\.locked$' | \ sed 's|\(\.[0-9a-f]\{40\}\)\{2\}\.[^\.\]\+\.locked$||' ) grep -vxF 'break_loops' "${work_dir}/build-list" | \ sort -k1,1 > \ "${work_dir}/build-list.sorted-by-package" for package in $( # shellcheck disable=SC2030 find "${work_dir}/build-list.loops/" -maxdepth 1 | \ grep '/loop_[0-9]\+$' | \ while read -r loop; do if [ -z "$( ( cat "${loop}" echo "${locked_packages}" ) | \ sort | \ uniq -d )" ]; then cat "${loop}" fi done | \ sort | \ uniq -c | \ join -1 2 -2 1 -o 1.1,2.1,2.2,2.3,2.4 \ - \ "${work_dir}/build-list.sorted-by-package" | \ while read -r count package git_revision git_mod_revision repository; do if [ -f "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken" ]; then trials=$( wc -l < \ "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken" ) else trials='0' fi printf '%s %s %s.%s.%s.%s\n' \ "${trials}" \ "${count}" \ "${package}" \ "${git_revision}" \ "${git_mod_revision}" \ "${repository}" done | \ sort -k1n,1 -k2nr,2 | \ cut -d' ' -f3 ); do # shellcheck disable=SC2046,SC2031 hand_out_assignment $( echo "${package}" | \ sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|' ) done # Remove the lock file >&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready' exit 2