lib/common-functions: recompress_gz() new to recompress databases before sending, so they are transferred faster by rsync

5 files changed, 38 insertions, 2 deletions

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index a60f058..8625d21 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -176,6 +176,10 @@ if [ -s "${tmp_dir}/sftp-command" ]; then
 fi
 
 while read -r repo_arch; do
+  recompress_gz \
+    "${tmp_dir}" \
+    "${tmp_dir}/${repo_arch}/build-support."*".tar.gz" \
+    "${tmp_dir}/${repo_arch}/build-support."*".tar.gz.old"
   ${master_mirror_rsync_command} \
     "${tmp_dir}/${repo_arch}/build-support.db."* \
     "${tmp_dir}/${repo_arch}/build-support.files."* \
diff --git a/bin/db-update b/bin/db-update
index 7d5af50..a8659a7 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -418,6 +418,10 @@ for source_stability in \
 
   # and push our local *.db.tar.gz via rsync
   while read -r arch repo; do
+    recompress_gz \
+      "${tmp_dir}" \
+      "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
+      "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
     # shellcheck disable=SC2086
     ${master_mirror_rsync_command} \
       "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
diff --git a/bin/delete-packages b/bin/delete-packages
index 4767d00..8c515c2 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -288,9 +288,13 @@ while read -r arch repo; do
         cut -d' ' -f3
     )
   if ! ${no_action}; then
+    recompress_gz \
+      "${tmp_dir}" \
+      "${tmp_dir}/repos/${arch}/${repo}."*".tar.gz" \
+      "${tmp_dir}/repos/${arch}/${repo}."*".tar.gz.old"
     ${master_mirror_rsync_command} \
-      "${tmp_dir}/repos/${arch}/${repo}.db.tar.gz" \
-      "${tmp_dir}/repos/${arch}/${repo}.files.tar.gz" \
+      "${tmp_dir}/repos/${arch}/${repo}.db."* \
+      "${tmp_dir}/repos/${arch}/${repo}.files."* \
       "${master_mirror_rsync_directory}/${arch}/${repo}/"
   fi
 done < \
diff --git a/bin/return-assignment b/bin/return-assignment
index 2e0d7db..1eba7ac 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -576,6 +576,10 @@ find . \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -printf '%f\n' | \
 cut -d' ' -f4,5 "${tmp_dir}/repository-ids" | \
   sort -u | \
   while read -r arch repo; do
+    recompress_gz \
+      "${tmp_dir}" \
+      "${arch}/${repo}/${repo}."*".tar.gz" \
+      "${arch}/${repo}/${repo}."*".tar.gz.old"
     # shellcheck disable=SC2046
     ${master_mirror_rsync_command} \
       "${arch}/${repo}/${repo}.db."* \
diff --git a/lib/common-functions b/lib/common-functions
index d8da4a7..aa78ebd 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -758,3 +758,23 @@ verbose_flock() {
     return ${err}
   }
 }
+
+# recompress_gz $tmp_dir $file1.gz $file2.gz ...
+# recompress the given file(s) to make them rsync friendly
+recompress_gz() {
+  tmp_file=$(
+    mktemp "$1/recompress_gz.XXXXXXXX)"
+  )
+  shift
+  local file
+  for file in "$@"; do
+    if [ ! -f "${file}" ]; then
+      continue
+    fi
+    mv "${file}" "${tmp_file}"
+    zcat "${tmp_file}" | \
+      gzip --best --rsyncable > \
+      "${file}"
+  done
+  rm "${tmp_file}"
+}