From e040271149a8ca0a7d8cb9322e60ac91f3fcb21a Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 31 Jan 2020 10:52:26 +0100 Subject: bin/db-update: uses intentions-queue, now --- bin/db-update | 234 +++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 150 insertions(+), 84 deletions(-) (limited to 'bin/db-update') diff --git a/bin/db-update b/bin/db-update index e1255d6..e6bb1b0 100755 --- a/bin/db-update +++ b/bin/db-update @@ -9,8 +9,6 @@ # TODO: make (all) locking less restrictive -# TODO: use the intention-queue - # shellcheck disable=SC2039,SC2119,SC2120 # shellcheck source=../lib/load-configuration @@ -67,7 +65,7 @@ eval set -- "$( block_flag='-n' ignore_insanity=false -no_action=false +no_action_flag='' progressive=false force_pkgs='' only='' @@ -90,7 +88,7 @@ do ignore_insanity=true ;; -n|--no-action) - no_action=true + no_action_flag='-n' ;; -o|--only) shift @@ -167,7 +165,7 @@ if [ -n "${only}" ]; then ) fi -if ! ${no_action}; then +if [ -z "${no_action_flag}" ]; then # Create lock. exec 9> "${package_database_lock_file}" if ! verbose_flock ${block_flag} 9; then @@ -513,21 +511,35 @@ export TMPDIR="${tmp_dir}" "${tmp_dir}/rm" \ "${tmp_dir}/rm.id" + perma_tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.permanent.XXXXXXXXXX") # shellcheck disable=SC2086 for s in "${tmp_dir}/"*; do - sort -u "${s}" | \ - sponge "${s}" + sort -u "${s}" \ + > "${perma_tmp_dir}/${s##*/}" done # receive the repository databases from the master mirror - mkdir "${tmp_dir}/dbs" while read -r arch repo; do - mkdir -p "${tmp_dir}/dbs/${arch}/${repo}" - # shellcheck disable=SC2086 - failsafe_rsync \ - "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \ - "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \ - "${tmp_dir}/dbs/${arch}/${repo}/" + { + printf 'mkdir -p "%s/dbs/%s/%s"\n' \ + "${perma_tmp_dir}" \ + "${arch}" \ + "${repo}" + printf 'failsafe_rsync' + for suffix in 'db' 'files'; do + printf ' "%s/%s/%s/%s.%s."*' \ + "${master_mirror_rsync_directory}" \ + "${arch}" \ + "${repo}" \ + "${repo}" \ + "${suffix}" + done + printf ' "%s/dbs/%s/%s/"\n' \ + "${perma_tmp_dir}" \ + "${arch}" \ + "${repo}" + } \ + | intent_something "${no_action_flag}" done < \ "${tmp_dir}/repositories" @@ -537,11 +549,21 @@ export TMPDIR="${tmp_dir}" "${tmp_dir}/rm" | \ sort -u | \ while read -r arch repo; do - grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \ - sed ' + { +# TODO: sign repositories + printf 'repo-remove -q "%s/dbs/%s/%s/%s.db.tar.gz" ' \ + "${perma_tmp_dir}" \ + "${arch}" \ + "${repo}" \ + "${repo}" + grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" \ + | sed ' s/\(-[^-]\+\)\{3\} \S\+ \S\+$// - ' | \ - xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz" + ' \ + | tr '\n' ' ' + printf '\n' + } \ + | intent_something "${no_action_flag}" done # copy and delete moved packages @@ -550,89 +572,133 @@ export TMPDIR="${tmp_dir}" "${tmp_dir}/mv" | \ sort -u | \ while read -r from_arch from_repo to_arch to_repo; do - grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \ - sed ' + { + printf '"%s/bin/repo-copy" ' \ + "${base_dir}" + printf '"%s/dbs/%s/%s/%s.db.tar.gz" ' \ + "${perma_tmp_dir}" "${from_arch}" "${from_repo}" "${from_repo}" \ + "${perma_tmp_dir}" "${to_arch}" "${to_repo}" "${to_repo}" + grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" \ + | sed ' s/-[^-]\+\( \S\+\)\{4\}$// - ' | \ - xargs -r "${base_dir}/bin/repo-copy" \ - "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \ - "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz" - grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \ - sed ' + ' \ + | tr '\n' ' ' + printf '\n' + printf 'repo-remove -q "%s/dbs/%s/%s/%s.db.tar.gz" ' \ + "${perma_tmp_dir}" \ + "${from_arch}" \ + "${from_repo}" \ + "${from_repo}" + grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" \ + | sed ' s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$// - ' | \ - xargs -r repo-remove -q \ - "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" + ' \ + | tr '\n' ' ' + printf '\n' + } \ + | intent_something "${no_action_flag}" done # create real file names of packages, because # mysql_query_and_delete_unneeded_binary_packages does so, too - sed -i ' - s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1, - ' "${tmp_dir}/rm" + { + printf 'sed -i '"'"'\n' + printf ' s,^\\(\\S\\+\) \\(\\S\\+\\) \\(\\S\\+\\)$,\\2/\\3/\\1,\n' + printf "'"' "%s/rm"\n' \ + "${perma_tmp_dir}" + } \ + | intent_something "${no_action_flag}" # somewhat inaccurate - if ! ${no_action}; then - { - printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n' - printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' - printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \ - "${tmp_dir}/mv.id" 'moved_bpir' \ - "${tmp_dir}/rm.id" 'replaced_bpir' - printf 'DELETE `binary_packages_in_repositories`' - printf ' FROM `binary_packages_in_repositories`' - printf ' JOIN `replaced_bpir`' - printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n' - mysql_query_and_delete_unneeded_binary_packages - printf 'UPDATE `binary_packages_in_repositories`' - printf ' JOIN `moved_bpir`' - printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`' - printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,' - printf '`binary_packages_in_repositories`.`last_moved`=NOW()' - printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n' - } | \ - mysql_run_query | \ - sort -u >> \ - "${tmp_dir}/rm" - fi + { + printf '{\n' + printf ' mysql_run_query' + printf ' | sort -u' + printf ' >> "%s/rm"\n' \ + "${perma_tmp_dir}" + printf '} <