#!/bin/sh # move binary packages from staging to testing (if possible [1]) and # additionally tested packages from testing to the respective stable # repository (if possible [1]) # The condition [1] is explained in the stored function # calculate_maximal_moveable_set which is created in bin/bootsrap-mysql # TODO: separate locks for staging, testing (and stable) # TODO: sub_pkgrel=0 may or may not be omitted for the actual package :-/ # TODO: keep .testing, .tested, .done in sync # shellcheck disable=SC2039 # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" # shellcheck disable=SC2016 usage() { >&2 echo '' >&2 echo 'db-update [options] [packages]:' >&2 echo ' move tested packages from testing to stable.' >&2 echo ' move possible packages from staging to testing.' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -b|--block: If necessary, wait for lock blocking.' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -n|--no-action: Only print what would be moved.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( getopt -o bhn \ --long block \ --long help \ --long no-action \ -n "$(basename "$0")" -- "$@" || \ echo usage )" block_flag='-n' no_action=false while true do case "$1" in -b|--block) block_flag='' ;; -h|--help) usage 0 ;; -n|--no-action) no_action=true ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done if [ $# -ne 0 ]; then >&2 echo 'db-update: too many arguments' usage fi if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit fi # Create tmp_dir, lock and trap. tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX") trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT exec 9> "${package_database_lock_file}" if ! flock ${block_flag} 9; then >&2 echo 'come back (shortly) later - I cannot lock package database.' exit 0 fi exec 8> "${sanity_check_lock_file}" if ! flock -s ${block_flag} 8; then >&2 echo 'come back (shortly) later - sanity-check currently running.' exit 0 fi # shellcheck disable=SC2119 mysql_cleanup for source_stability in 'testing' 'staging'; do find "${tmp_dir}" -mindepth 1 -delete # shellcheck disable=SC2016 { printf 'CALL calculate_maximal_moveable_set("%s");\n' \ "${source_stability}" printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n' printf 'INSERT IGNORE INTO `rps` (`id`)' printf ' SELECT `moveable_binary_packages`.`to_repository`' printf ' FROM `moveable_binary_packages`;\n' printf 'INSERT IGNORE INTO `rps` (`id`)' printf ' SELECT `binary_packages`.`repository`' printf ' FROM `moveable_binary_packages`' printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`;\n' printf 'INSERT IGNORE INTO `rps` (`id`)' printf ' SELECT `binary_packages`.`repository`' printf ' FROM `replaced_binary_packages`' printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`;\n' printf 'SELECT "repositories",`repositories`.`name`' printf ' FROM `repositories`' printf ' JOIN `rps` ON `rps`.`id`=`repositories`.`id`;\n' printf 'SELECT "mv.id",`moveable_binary_packages`.`id`,`moveable_binary_packages`.`to_repository`' printf ' FROM `moveable_binary_packages`;\n' printf 'SELECT "mv",' mysql_package_name_query printf ',`repositories`.`name`,`new_repo`.`name`' printf ' FROM `moveable_binary_packages`' printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`' mysql_join_binary_packages_repositories mysql_join_binary_packages_architectures printf ' JOIN `repositories` AS `new_repo` ON `new_repo`.`id`=`moveable_binary_packages`.`to_repository`' printf ';\n' printf 'SELECT "rm.id",`replaced_binary_packages`.`id`' printf ' FROM `replaced_binary_packages`;\n' printf 'SELECT "rm",' mysql_package_name_query printf ',`repositories`.`name`' printf ' FROM `replaced_binary_packages`' printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`' mysql_join_binary_packages_repositories mysql_join_binary_packages_architectures printf ';\n' } | \ mysql_run_query | \ tr '\t' ' ' | \ grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \ while read -r what content; do printf '%s\n' "${content}" >> \ "${tmp_dir}/${what}" done if [ ! -s "${tmp_dir}/repositories" ]; then >&2 echo 'Nothing to move from %s.' "${source_stability}" continue fi # shellcheck disable=SC2086 for s in "${tmp_dir}/"*; do sort -u "${s}" | \ sponge "${s}" done # receive the repository databases from the master mirror mkdir "${tmp_dir}/dbs" while read -r repo; do mkdir "${tmp_dir}/dbs/${repo}" # shellcheck disable=SC2086 ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \ "${tmp_dir}/dbs/${repo}/" tar -Oxzf "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz" --wildcards '*/desc' | \ sed -n ' /^%FILENAME%$/{ N s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/ T p } ' done < \ "${tmp_dir}/repositories" | \ while read -r old new; do for file in 'rm' 'mv'; do sed -i ' s/\(\s\|^\)'"$(str_to_regex "${old}")"'\(\s\|$\)/\1'"${new}"'\2/ ' "${tmp_dir}/${file}" done done # remove to-be-deleted packages while read -r pkgname repo; do cd "${tmp_dir}/dbs/${repo}" repo-remove -q "${repo}.db.tar.gz" "${pkgname%-*-*-*}" done < \ "${tmp_dir}/rm" # delete and add moved packages mkdir "${tmp_dir}/transit" while read -r pkgname from_repo to_repo; do ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/${from_repo}/${pkgname}" \ "${master_mirror_rsync_directory}/i686/${from_repo}/${pkgname}.sig" \ "${tmp_dir}/transit/" cd "${tmp_dir}/dbs/${from_repo}" repo-remove -q "${from_repo}.db.tar.gz" "${pkgname%-*-*-*}" cd "${tmp_dir}/dbs/${to_repo}" repo-add -q "${to_repo}.db.tar.gz" "${tmp_dir}/transit/${pkgname}" rm \ "${tmp_dir}/transit/${pkgname}" \ "${tmp_dir}/transit/${pkgname}.sig" done < \ "${tmp_dir}/mv" # move the packages remotely via sftp { sed ' s,^\(\S\+\) \(\S\+\)$,rm "i686/\2/\1"\nrm "i686/\2/\1.sig", ' "${tmp_dir}/rm" sed ' s,^\(\S\+\) \(\S\+\) \(\S\+\)$,rename "i686/\2/\1" "i686/\3/\1"\nrename "i686/\2/\1.sig" "i686/\3/\1.sig", ' "${tmp_dir}/mv" echo 'quit' } | \ if ${no_action}; then sed 's|^|sftp: |' else ${master_mirror_sftp_command} fi if ${no_action}; then continue fi # and push our local *.db.tar.gz via rsync while read -r repo; do # shellcheck disable=SC2086 ${master_mirror_rsync_command} \ "${tmp_dir}/dbs/${repo}/${repo}.db."* \ "${tmp_dir}/dbs/${repo}/${repo}.files."* \ "${master_mirror_rsync_directory}/i686/${repo}/" done < \ "${tmp_dir}/repositories" trigger_mirror_refreshs # shellcheck disable=SC2016 { printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n' printf 'CREATE TEMPORARY TABLE `moved_binary_packages` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`;\n' \ "${tmp_dir}/mv.id" 'moved_binary_packages' \ "${tmp_dir}/rm.id" 'replaced_binary_packages' printf 'DELETE `binary_packages` FROM `binary_packages`' printf ' JOIN `replaced_binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`;\n' printf 'UPDATE `binary_packages`' printf ' JOIN `moved_binary_packages` ON `binary_packages`.`id`=`moved_binary_packages`.`id`' printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n' } | \ mysql_run_query done