diff options
| -rwxr-xr-x | bin/build-master-status | 27 | ||||
| -rwxr-xr-x | bin/common-functions | 28 | ||||
| -rwxr-xr-x | bin/db-update | 196 | ||||
| -rwxr-xr-x | bin/return-assignment | 23 | ||||
| -rwxr-xr-x | conf/default.conf | 4 |
5 files changed, 198 insertions, 80 deletions
diff --git a/bin/build-master-status b/bin/build-master-status index 84e5dc6..ab052be 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -5,8 +5,13 @@ . "${0%/*}/../conf/default.conf" stable="$( - find "${master_mirror_directory}/i686/" \( -name '*testing' -o -name '*staging' \) -prune -o -name '*.pkg.tar.xz' -print 2> /dev/null | \ - wc -l + ${master_mirror_command} "${master_mirror_directory}/i686/" | \ + awk '{print $5}' | \ + grep -v 'testing$\|staging$\|^\.$' | \ + while read -r dir; do + ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/" + done | \ + grep -c '\.pkg\.tar\.xz$' )" tasks="$( wc -l < \ @@ -22,12 +27,22 @@ pending_packages="$( wc -l )" staging="$( - find "${master_mirror_directory}/i686/"*staging -name '*.pkg.tar.xz' 2> /dev/null | \ - wc -l + ${master_mirror_command} "${master_mirror_directory}/i686/" | \ + awk '{print $5}' | \ + grep 'staging$' | \ + while read -r dir; do + ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/" + done | \ + grep -c '\.pkg\.tar\.xz$' )" testing="$( - find "${master_mirror_directory}/i686/"*testing -name '*.pkg.tar.xz' 2> /dev/null | \ - wc -l + ${master_mirror_command} "${master_mirror_directory}/i686/" | \ + awk '{print $5}' | \ + grep 'testing$' | \ + while read -r dir; do + ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/" + done | \ + grep -c '\.pkg\.tar\.xz$' )" broken="$( ls "${work_dir}/package-states/" | \ diff --git a/bin/common-functions b/bin/common-functions index 911a580..cc59e94 100755 --- a/bin/common-functions +++ b/bin/common-functions @@ -106,7 +106,7 @@ find_repository_with_commit() { return 0 fi done - >&2 echo "can't find repository with commit '$1'" + >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1" exit 1 } @@ -299,18 +299,30 @@ official_or_community() { esac } -# remove_old_package_versions $package_file -# removes all other versions of $package_file in the current directory +# remove_old_package_versions $directory $package_file +# removes all other versions of $package_file in $directory on the master mirror remove_old_package_versions() { - local package="$1" + local directory="$1" + local package="$2" + local pkgname="${package%-*-*-*.pkg.tar.xz}" - ls | \ - grep "^$(str_to_regex "${pkgname}")\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \ - grep -v "^$(str_to_regex "${package}")\(\.sig\)\?\$" | \ - xargs -rn1 rm + ${master_mirror_command} \ + --recursive \ + --delete \ + $( \ + ${master_mirror_command} \ + "${master_mirror_directory}/${directory}/" | \ + awk '{print $5}' | \ + grep "^$(str_to_regex "${pkgname}")\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \ + grep -v "^$(str_to_regex "${package}")\(\.sig\)\?\$" | \ + sed 's|^|--include=|' + ) \ + '--exclude=*' \ + ./ \ + "${master_mirror_directory}/${directory}/" } diff --git a/bin/db-update b/bin/db-update index 5284c6a..1163633 100755 --- a/bin/db-update +++ b/bin/db-update @@ -30,6 +30,131 @@ usage() { [ -z "$1" ] && exit 1 || exit $1 } +# move_package $package $from_repository $to_repository +# the existence of a directory $tmp_dir is assumed + +move_package() { + + if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then + >&2 echo 'move_package: No tmp_dir provided.' + exit 2 + fi + + local package="$1" + local from_repo="$2" + local to_repo="$3" + local from_ending + local to_ending + local part + + if echo "${from_repo}" | \ + grep -q 'staging$' && \ + echo "${to_repo}" | \ + grep -q 'testing$'; then + from_ending='done' + to_ending='testing' + elif echo "${from_repo}" | \ + grep -q 'testing$' && \ + ! echo "${to_repo}" | \ + grep -q 'testing$\|staging$'; then + from_ending='testing' + to_ending='' + else + >&2 printf 'move_package: Cannot move package from "%s" to "%s".\n' "${from_repo}" "${to_repo}" + exit 2 + fi + + if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then + >&2 printf 'move_package: Cannot find package state file "%s"\n' "${package}.${from_ending}" + exit 2 + fi + + rm -rf --one-file-system "${tmp_dir}/"* + + cp \ + "${work_dir}/package-states/${package}.${from_ending}" \ + "${tmp_dir}/parts" + sed \ + 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \ + "${tmp_dir}/parts" > \ + "${tmp_dir}/parts_names" + sed \ + 'p;s|$|.sig|' \ + "${tmp_dir}/parts" > \ + "${tmp_dir}/parts_and_signatures" + + mkdir "${tmp_dir}/from" + mkdir "${tmp_dir}/to" + + ${master_mirror_command} \ + --files-from="${tmp_dir}/parts_and_signatures" \ + "${master_mirror_directory}/i686/${from_repo}/" \ + "${tmp_dir}/from/" + + ${master_mirror_command} \ + "${master_mirror_directory}/i686/${from_repo}/${from_repo}.db."* \ + "${tmp_dir}/from" + + ${master_mirror_command} \ + "${master_mirror_directory}/i686/${to_repo}/${to_repo}.db."* \ + "${tmp_dir}/to" + + repo-remove \ + "${tmp_dir}/from/${from_repo}.db.tar.gz" \ + $(cat "${tmp_dir}/parts_names") + + while read -r part; do + mv \ + "${tmp_dir}/from/${part}" \ + "${tmp_dir}/to/" + done < \ + "${tmp_dir}/parts_and_signatures" + + repo-add \ + "${tmp_dir}/to/${to_repo}.db.tar.gz" \ + $( + sed \ + "s|^|${tmp_dir}/to/|" \ + "${tmp_dir}/parts" + ) + + ${master_mirror_command} \ + --files-from="${tmp_dir}/parts_and_signatures" \ + "${tmp_dir}/to/" \ + "${master_mirror_directory}/i686/${to_repo}/" + + ${master_mirror_command} \ + "${tmp_dir}/to/${to_repo}.db."* \ + "${master_mirror_directory}/i686/${to_repo}/" + + ${master_mirror_command} \ + "${tmp_dir}/from/${from_repo}.db."* \ + "${master_mirror_directory}/i686/${from_repo}/" + + while read -r part; do + echo remove_old_package_versions "i686/${to_repo}" "${part}" + remove_old_package_versions "i686/${to_repo}" "${part}" + # the next line will remove _all_ versions of the package $part from $from_repo + echo remove_old_package_versions "i686/${from_repo}" "${part%.pkg.tar.xz}0.pkg.tar.xz" + remove_old_package_versions "i686/${from_repo}" "${part%.pkg.tar.xz}0.pkg.tar.xz" + done < \ + "${tmp_dir}/parts" + + if [ -z "${to_ending}" ]; then + rm \ + "${work_dir}/package-states/${package}.${from_ending}" + else + mv \ + "${work_dir}/package-states/${package}.${from_ending}" \ + "${work_dir}/package-states/${package}.${to_ending}" + fi + + updated_package_database=true + + rm -rf --one-file-system "${tmp_dir}/"* + +} + eval set -- "$( getopt -o bh \ --long block \ @@ -135,8 +260,9 @@ keep_packages="$( "${tmp_dir}/done_packages" )" printf '%s\n' "${keep_packages}" | \ - grep -vxF '' || true > \ - "${tmp_dir}/keep_packages" + grep -vxF '' > \ + "${tmp_dir}/keep_packages" || \ + true # find all dependencies of the unstageable packages mv \ @@ -200,20 +326,11 @@ if [ -z "${delete_packages}" ]; then rm -f "${build_list_lock_file}" flock -u 9 - rm -rf --one-file-system "${tmp_dir}" - clean_up_lock_file() { + rm -rf --one-file-system "${tmp_dir}" rm -f "${package_database_lock_file}" } -else - - rm -rf --one-file-system "${tmp_dir}" - - clean_up_lock_file() { - rm -f "${package_database_lock_file}" "${build_list_lock_file}" - } - fi # calculate unstageable packages from keep_packages and done_packages @@ -234,55 +351,28 @@ done_packages="$( unset keep_packages for package in ${done_packages}; do + if [ -z "${package}" ]; then continue fi - parts=$(cat "${work_dir}/package-states/${package}.done") - parts_names=$( - printf '%s\n' ${parts} | \ - sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' - ) + is_community="$(official_or_community "${package}")" - cd "${master_mirror_directory}/i686/${is_community}staging" - mkdir -p "../${is_community}testing" - repo-remove "${is_community}staging.db.tar.gz" ${parts_names} - for part in ${parts}; do - mv "${part}" "${part}.sig" "../${is_community}testing/" - updated_package_database=true - done - cd "${master_mirror_directory}/i686/${is_community}testing" - repo-add "${is_community}testing.db.tar.gz" ${parts} - for part in ${parts}; do - remove_old_package_versions "${part}" - done - - mv \ - "${work_dir}/package-states/${package}.done" \ - "${work_dir}/package-states/${package}.testing" + + move_package "${package}" "${is_community}staging" "${is_community}testing" + done +exit + # move packages in packages_to_stabilize from *testing/ to the stable repos for package in ${packages_to_stabilize}; do - parts=$(cat "${work_dir}/package-states/${package}.testing") - parts_names=$( - printf '%s\n' ${parts} | \ - sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' - ) - source="$(official_or_community ${package})testing" - destination="${package##*.}" - cd "${master_mirror_directory}/i686/${source}" - repo-remove "${source}.db.tar.gz" ${parts_names} - for part in ${parts}; do - mv "${part}" "${part}.sig" "../${destination}/" - updated_package_database=true - done - cd "${master_mirror_directory}/i686/${destination}" - repo-add "${destination}.db.tar.gz" ${parts} - for part in ${parts}; do - remove_old_package_versions "${part}" - done - - rm "${work_dir}/package-states/${package}.testing" + + if [ -z "${package}" ]; then + continue + fi + + move_package "${package}" "$(official_or_community ${package})testing" "${package##*.}" + done if [ -n "${delete_packages}" ]; then diff --git a/bin/return-assignment b/bin/return-assignment index 78d888a..5252518 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -144,22 +144,21 @@ fi # move packages destination="$(official_or_community "$1.$2.$3.$4")staging" -mkdir -p "${master_mirror_directory}/i686/${destination}/" -mv \ +${master_mirror_command} \ + "${master_mirror_directory}/i686/${destination}/${destination}.db.*" \ + . +repo-add "${destination}.db.tar.gz" ${packages} +# repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages} + +${master_mirror_command} \ + "${destination}.db."* *".pkg.tar.xz" \ *".pkg.tar.xz.sig" \ "${master_mirror_directory}/i686/${destination}/" -( - cd "${master_mirror_directory}/i686/${destination}" - repo-add "${destination}.db.tar.gz" ${packages} - # repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages} - - # remove old versions of same packages - for package in ${packages}; do - remove_old_package_versions "${package}" - done -) +for package in ${packages}; do + remove_old_package_versions "i686/${destination}" "${package}" +done # remove old state files (these should be only "done" markers, but # actually we don't care what it is) diff --git a/conf/default.conf b/conf/default.conf index 0e9dc6e..e4e3fc4 100755 --- a/conf/default.conf +++ b/conf/default.conf @@ -30,7 +30,9 @@ max_parallel_build_per_client=2 repo_key='0xdeadbeef' package_key='0x15eebadc0de' -master_mirror_directory='/path/to/master/mirror' + +master_mirror_command='rsync --password-file=/home/slave/rsync.password' +master_mirror_directory='rsync://buildmaster@mirror.archlinux32.org/packages32' # possibly pull in custom modifications |