From e4677e04f39bfca06ca7b1fb92f89b7684f20e87 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 19 Jun 2017 13:47:32 +0200
Subject: switch from sshfs to rsync - to avoid needing to remount

---
 bin/build-master-status |  27 +++++--
 bin/common-functions    |  28 +++++--
 bin/db-update           | 196 +++++++++++++++++++++++++++++++++++-------------
 bin/return-assignment   |  23 +++---
 conf/default.conf       |   4 +-
 5 files changed, 198 insertions(+), 80 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 84e5dc6..ab052be 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -5,8 +5,13 @@
 . "${0%/*}/../conf/default.conf"
 
 stable="$(
-  find "${master_mirror_directory}/i686/" \( -name '*testing' -o -name '*staging' \) -prune -o -name '*.pkg.tar.xz' -print 2> /dev/null | \
-    wc -l
+  ${master_mirror_command} "${master_mirror_directory}/i686/" | \
+    awk '{print $5}' | \
+    grep -v 'testing$\|staging$\|^\.$' | \
+    while read -r dir; do
+      ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/"
+    done | \
+    grep -c '\.pkg\.tar\.xz$'
   )"
 tasks="$(
   wc -l < \
@@ -22,12 +27,22 @@ pending_packages="$(
     wc -l
 )"
 staging="$(
-  find "${master_mirror_directory}/i686/"*staging -name '*.pkg.tar.xz' 2> /dev/null | \
-    wc -l
+  ${master_mirror_command} "${master_mirror_directory}/i686/" | \
+    awk '{print $5}' | \
+    grep 'staging$' | \
+    while read -r dir; do
+      ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/"
+    done | \
+    grep -c '\.pkg\.tar\.xz$'
   )"
 testing="$(
-  find "${master_mirror_directory}/i686/"*testing -name '*.pkg.tar.xz' 2> /dev/null | \
-    wc -l
+  ${master_mirror_command} "${master_mirror_directory}/i686/" | \
+    awk '{print $5}' | \
+    grep 'testing$' | \
+    while read -r dir; do
+      ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/"
+    done | \
+    grep -c '\.pkg\.tar\.xz$'
   )"
 broken="$(
   ls "${work_dir}/package-states/" | \
diff --git a/bin/common-functions b/bin/common-functions
index 911a580..cc59e94 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -106,7 +106,7 @@ find_repository_with_commit() {
       return 0
     fi
   done
-  >&2 echo "can't find repository with commit '$1'"
+  >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1"
   exit 1
 
 }
@@ -299,18 +299,30 @@ official_or_community() {
   esac
 }
 
-# remove_old_package_versions $package_file
-# removes all other versions of $package_file in the current directory
+# remove_old_package_versions $directory $package_file
+# removes all other versions of $package_file in $directory on the master mirror
 
 remove_old_package_versions() {
 
-  local package="$1"
+  local directory="$1"
+  local package="$2"
+
   local pkgname="${package%-*-*-*.pkg.tar.xz}"
 
-  ls | \
-    grep "^$(str_to_regex "${pkgname}")\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \
-    grep -v "^$(str_to_regex "${package}")\(\.sig\)\?\$" | \
-    xargs -rn1 rm
+  ${master_mirror_command} \
+    --recursive \
+    --delete \
+    $( \
+      ${master_mirror_command} \
+        "${master_mirror_directory}/${directory}/" | \
+        awk '{print $5}' | \
+        grep "^$(str_to_regex "${pkgname}")\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \
+        grep -v "^$(str_to_regex "${package}")\(\.sig\)\?\$" | \
+        sed 's|^|--include=|'
+    ) \
+    '--exclude=*' \
+    ./ \
+    "${master_mirror_directory}/${directory}/"
 
 }
 
diff --git a/bin/db-update b/bin/db-update
index 5284c6a..1163633 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -30,6 +30,131 @@ usage() {
   [ -z "$1" ] && exit 1 || exit $1
 }
 
+# move_package $package $from_repository $to_repository
+# the existence of a directory $tmp_dir is assumed
+
+move_package() {
+
+  if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
+    >&2 echo 'move_package: No tmp_dir provided.'
+    exit 2
+  fi
+
+  local package="$1"
+  local from_repo="$2"
+  local to_repo="$3"
+  local from_ending
+  local to_ending
+  local part
+
+  if echo "${from_repo}" | \
+    grep -q 'staging$' && \
+    echo "${to_repo}" | \
+      grep -q 'testing$'; then
+    from_ending='done'
+    to_ending='testing'
+  elif echo "${from_repo}" | \
+    grep -q 'testing$' && \
+    ! echo "${to_repo}" | \
+      grep -q 'testing$\|staging$'; then
+    from_ending='testing'
+    to_ending=''
+  else
+    >&2 printf 'move_package: Cannot move package from "%s" to "%s".\n' "${from_repo}" "${to_repo}"
+    exit 2
+  fi
+
+  if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
+    >&2 printf 'move_package: Cannot find package state file "%s"\n' "${package}.${from_ending}"
+    exit 2
+  fi
+
+  rm -rf --one-file-system "${tmp_dir}/"*
+
+  cp \
+    "${work_dir}/package-states/${package}.${from_ending}" \
+    "${tmp_dir}/parts"
+  sed \
+    's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
+    "${tmp_dir}/parts" > \
+    "${tmp_dir}/parts_names"
+  sed \
+    'p;s|$|.sig|' \
+    "${tmp_dir}/parts" > \
+    "${tmp_dir}/parts_and_signatures"
+
+  mkdir "${tmp_dir}/from"
+  mkdir "${tmp_dir}/to"
+
+  ${master_mirror_command} \
+    --files-from="${tmp_dir}/parts_and_signatures" \
+    "${master_mirror_directory}/i686/${from_repo}/" \
+    "${tmp_dir}/from/"
+
+  ${master_mirror_command} \
+    "${master_mirror_directory}/i686/${from_repo}/${from_repo}.db."* \
+    "${tmp_dir}/from"
+
+  ${master_mirror_command} \
+    "${master_mirror_directory}/i686/${to_repo}/${to_repo}.db."* \
+    "${tmp_dir}/to"
+
+  repo-remove \
+    "${tmp_dir}/from/${from_repo}.db.tar.gz" \
+    $(cat "${tmp_dir}/parts_names")
+
+  while read -r part; do
+    mv \
+      "${tmp_dir}/from/${part}" \
+      "${tmp_dir}/to/"
+  done < \
+    "${tmp_dir}/parts_and_signatures"
+
+  repo-add \
+    "${tmp_dir}/to/${to_repo}.db.tar.gz" \
+    $(
+      sed \
+        "s|^|${tmp_dir}/to/|" \
+        "${tmp_dir}/parts"
+    )
+
+  ${master_mirror_command} \
+    --files-from="${tmp_dir}/parts_and_signatures" \
+    "${tmp_dir}/to/" \
+    "${master_mirror_directory}/i686/${to_repo}/"
+
+  ${master_mirror_command} \
+    "${tmp_dir}/to/${to_repo}.db."* \
+    "${master_mirror_directory}/i686/${to_repo}/"
+
+  ${master_mirror_command} \
+    "${tmp_dir}/from/${from_repo}.db."* \
+    "${master_mirror_directory}/i686/${from_repo}/"
+
+  while read -r part; do
+    echo remove_old_package_versions "i686/${to_repo}" "${part}"
+    remove_old_package_versions "i686/${to_repo}" "${part}"
+    # the next line will remove _all_ versions of the package $part from $from_repo
+    echo remove_old_package_versions "i686/${from_repo}" "${part%.pkg.tar.xz}0.pkg.tar.xz"
+    remove_old_package_versions "i686/${from_repo}" "${part%.pkg.tar.xz}0.pkg.tar.xz"
+  done < \
+    "${tmp_dir}/parts"
+
+  if [ -z "${to_ending}" ]; then
+    rm \
+      "${work_dir}/package-states/${package}.${from_ending}"
+  else
+    mv \
+      "${work_dir}/package-states/${package}.${from_ending}" \
+      "${work_dir}/package-states/${package}.${to_ending}"
+  fi
+
+  updated_package_database=true
+
+  rm -rf --one-file-system "${tmp_dir}/"*
+
+}
+
 eval set -- "$(
   getopt -o bh \
     --long block \
@@ -135,8 +260,9 @@ keep_packages="$(
     "${tmp_dir}/done_packages"
 )"
 printf '%s\n' "${keep_packages}" | \
-  grep -vxF '' || true > \
-  "${tmp_dir}/keep_packages"
+  grep -vxF '' > \
+  "${tmp_dir}/keep_packages" || \
+  true
 
 # find all dependencies of the unstageable packages
 mv \
@@ -200,20 +326,11 @@ if [ -z "${delete_packages}" ]; then
   rm -f "${build_list_lock_file}"
   flock -u 9
 
-  rm -rf --one-file-system "${tmp_dir}"
-
   clean_up_lock_file() {
+    rm -rf --one-file-system "${tmp_dir}"
     rm -f "${package_database_lock_file}"
   }
 
-else
-
-  rm -rf --one-file-system "${tmp_dir}"
-
-  clean_up_lock_file() {
-    rm -f "${package_database_lock_file}" "${build_list_lock_file}"
-  }
-
 fi
 
 # calculate unstageable packages from keep_packages and done_packages
@@ -234,55 +351,28 @@ done_packages="$(
 unset keep_packages
 
 for package in ${done_packages}; do
+
   if [ -z "${package}" ]; then
     continue
   fi
-  parts=$(cat "${work_dir}/package-states/${package}.done")
-  parts_names=$(
-    printf '%s\n' ${parts} | \
-      sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||'
-  )
+
   is_community="$(official_or_community "${package}")"
-  cd "${master_mirror_directory}/i686/${is_community}staging"
-  mkdir -p "../${is_community}testing"
-  repo-remove "${is_community}staging.db.tar.gz" ${parts_names}
-  for part in ${parts}; do
-    mv "${part}" "${part}.sig" "../${is_community}testing/"
-    updated_package_database=true
-  done
-  cd "${master_mirror_directory}/i686/${is_community}testing"
-  repo-add "${is_community}testing.db.tar.gz" ${parts}
-  for part in ${parts}; do
-    remove_old_package_versions "${part}"
-  done
-
-  mv \
-    "${work_dir}/package-states/${package}.done" \
-    "${work_dir}/package-states/${package}.testing"
+
+  move_package "${package}" "${is_community}staging" "${is_community}testing"
+
 done
 
+exit
+
 # move packages in packages_to_stabilize from *testing/ to the stable repos
 for package in ${packages_to_stabilize}; do
-  parts=$(cat "${work_dir}/package-states/${package}.testing")
-  parts_names=$(
-    printf '%s\n' ${parts} | \
-      sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||'
-  )
-  source="$(official_or_community ${package})testing"
-  destination="${package##*.}"
-  cd "${master_mirror_directory}/i686/${source}"
-  repo-remove "${source}.db.tar.gz" ${parts_names}
-  for part in ${parts}; do
-    mv "${part}" "${part}.sig" "../${destination}/"
-    updated_package_database=true
-  done
-  cd "${master_mirror_directory}/i686/${destination}"
-  repo-add "${destination}.db.tar.gz" ${parts}
-  for part in ${parts}; do
-    remove_old_package_versions "${part}"
-  done
-
-  rm "${work_dir}/package-states/${package}.testing"
+
+  if [ -z "${package}" ]; then
+    continue
+  fi
+
+  move_package "${package}" "$(official_or_community ${package})testing" "${package##*.}"
+
 done
 
 if [ -n "${delete_packages}" ]; then
diff --git a/bin/return-assignment b/bin/return-assignment
index 78d888a..5252518 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -144,22 +144,21 @@ fi
 # move packages
 destination="$(official_or_community "$1.$2.$3.$4")staging"
 
-mkdir -p "${master_mirror_directory}/i686/${destination}/"
-mv \
+${master_mirror_command} \
+  "${master_mirror_directory}/i686/${destination}/${destination}.db.*" \
+  .
+repo-add "${destination}.db.tar.gz" ${packages}
+# repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages}
+
+${master_mirror_command} \
+  "${destination}.db."*
   *".pkg.tar.xz" \
   *".pkg.tar.xz.sig" \
   "${master_mirror_directory}/i686/${destination}/"
 
-(
-  cd "${master_mirror_directory}/i686/${destination}"
-  repo-add "${destination}.db.tar.gz" ${packages}
-  # repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages}
-
-  # remove old versions of same packages
-  for package in ${packages}; do
-    remove_old_package_versions "${package}"
-  done
-)
+for package in ${packages}; do
+  remove_old_package_versions "i686/${destination}" "${package}"
+done
 
 # remove old state files (these should be only "done" markers, but
 # actually we don't care what it is)
diff --git a/conf/default.conf b/conf/default.conf
index 0e9dc6e..e4e3fc4 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -30,7 +30,9 @@ max_parallel_build_per_client=2
 
 repo_key='0xdeadbeef'
 package_key='0x15eebadc0de'
-master_mirror_directory='/path/to/master/mirror'
+
+master_mirror_command='rsync --password-file=/home/slave/rsync.password'
+master_mirror_directory='rsync://buildmaster@mirror.archlinux32.org/packages32'
 
 # possibly pull in custom modifications
 
-- 
cgit v1.2.3-54-g00ecf