summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2018-06-13 16:22:08 +0200
committerErich Eckner <git@eckner.net>2018-06-13 16:22:08 +0200
commit1433976802baf67c89038d699d0a50522bc138cc (patch)
tree10c3b0adb34c644fdbeaf20a2a09b925c495987c
parentd7d699efe7941f626617e952fb8ecf057f493ec8 (diff)
downloadbuilder-1433976802baf67c89038d699d0a50522bc138cc.tar.xz
lib/common-functions: recompress_gz() new to recompress databases before sending, so they are transferred faster by rsync
-rwxr-xr-xbin/copy-to-build-support4
-rwxr-xr-xbin/db-update4
-rwxr-xr-xbin/delete-packages8
-rwxr-xr-xbin/return-assignment4
-rwxr-xr-xlib/common-functions20
5 files changed, 38 insertions, 2 deletions
diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index a60f058..8625d21 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -176,6 +176,10 @@ if [ -s "${tmp_dir}/sftp-command" ]; then
fi
while read -r repo_arch; do
+ recompress_gz \
+ "${tmp_dir}" \
+ "${tmp_dir}/${repo_arch}/build-support."*".tar.gz" \
+ "${tmp_dir}/${repo_arch}/build-support."*".tar.gz.old"
${master_mirror_rsync_command} \
"${tmp_dir}/${repo_arch}/build-support.db."* \
"${tmp_dir}/${repo_arch}/build-support.files."* \
diff --git a/bin/db-update b/bin/db-update
index 7d5af50..a8659a7 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -418,6 +418,10 @@ for source_stability in \
# and push our local *.db.tar.gz via rsync
while read -r arch repo; do
+ recompress_gz \
+ "${tmp_dir}" \
+ "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
+ "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
# shellcheck disable=SC2086
${master_mirror_rsync_command} \
"${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
diff --git a/bin/delete-packages b/bin/delete-packages
index 4767d00..8c515c2 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -288,9 +288,13 @@ while read -r arch repo; do
cut -d' ' -f3
)
if ! ${no_action}; then
+ recompress_gz \
+ "${tmp_dir}" \
+ "${tmp_dir}/repos/${arch}/${repo}."*".tar.gz" \
+ "${tmp_dir}/repos/${arch}/${repo}."*".tar.gz.old"
${master_mirror_rsync_command} \
- "${tmp_dir}/repos/${arch}/${repo}.db.tar.gz" \
- "${tmp_dir}/repos/${arch}/${repo}.files.tar.gz" \
+ "${tmp_dir}/repos/${arch}/${repo}.db."* \
+ "${tmp_dir}/repos/${arch}/${repo}.files."* \
"${master_mirror_rsync_directory}/${arch}/${repo}/"
fi
done < \
diff --git a/bin/return-assignment b/bin/return-assignment
index 2e0d7db..1eba7ac 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -576,6 +576,10 @@ find . \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -printf '%f\n' | \
cut -d' ' -f4,5 "${tmp_dir}/repository-ids" | \
sort -u | \
while read -r arch repo; do
+ recompress_gz \
+ "${tmp_dir}" \
+ "${arch}/${repo}/${repo}."*".tar.gz" \
+ "${arch}/${repo}/${repo}."*".tar.gz.old"
# shellcheck disable=SC2046
${master_mirror_rsync_command} \
"${arch}/${repo}/${repo}.db."* \
diff --git a/lib/common-functions b/lib/common-functions
index d8da4a7..aa78ebd 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -758,3 +758,23 @@ verbose_flock() {
return ${err}
}
}
+
+# recompress_gz $tmp_dir $file1.gz $file2.gz ...
+# recompress the given file(s) to make them rsync friendly
+recompress_gz() {
+ tmp_file=$(
+ mktemp "$1/recompress_gz.XXXXXXXX)"
+ )
+ shift
+ local file
+ for file in "$@"; do
+ if [ ! -f "${file}" ]; then
+ continue
+ fi
+ mv "${file}" "${tmp_file}"
+ zcat "${tmp_file}" | \
+ gzip --best --rsyncable > \
+ "${file}"
+ done
+ rm "${tmp_file}"
+}