diff options
| -rwxr-xr-x | bin/bootstrap-mysql | 2 | ||||
| -rwxr-xr-x | bin/repo-copy | 92 | ||||
| -rwxr-xr-x | bin/sanity-check | 9 | ||||
| -rwxr-xr-x | lib/mysql-functions | 75 |
4 files changed, 159 insertions, 19 deletions
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 8b01c38..afe1850 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -32,6 +32,8 @@ done | \ sort -k2,2 > \ "${tmp_dir}/master-mirror-listing" +# TODO: include hash of binary package in mysql database + if [ ! "$1" = 'slim' ]; then tables=$( printf '%s\n' \ diff --git a/bin/repo-copy b/bin/repo-copy new file mode 100755 index 0000000..8566d7c --- /dev/null +++ b/bin/repo-copy @@ -0,0 +1,92 @@ +#!/bin/sh + +# Rudimentary copy a package from one repository to another. +# Note, that we do _not_ need to have the package itself, since all +# relevant information is already in the original package database. + +# "Rudimentary" means the following restrictions: +# - no arguments are accepted +# - no database signatures are handled +# - only *.db.tar.gz and *.files.tar.gz are recognized as database + +usage() { + >&2 echo 'usage:' + >&2 echo ' repo-copy from-repo.db.tar.gz to-repo.db.tar.gz package1 package2 ...' + >&2 echo + >&2 echo 'Note, that the packages must be given with version, e.g. "linux-4.15.7-1.0".' + exit 2 +} + +if [ $# -le 2 ]; then + usage +fi + +from_repo="$1" +to_repo="$2" +shift +shift + +tmp_dir=$(mktemp -d) +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + +# extract the databases +for repo in 'from' 'to'; do + for archive in 'db' 'files'; do + eval 'repo_db="${'"${repo}"'_repo}"' + if [ "${repo_db}" = "${repo_db%.db.tar.gz}" ]; then + >&2 printf '"%s" has an invalid suffix.\n' "${repo_db}" + usage + fi + if [ "${archive}" = 'files' ]; then + repo_db="${repo_db%.db.tar.gz}.files.tar.gz" + fi + if [ ! -f "${repo_db}" ]; then + >&2 printf 'Cannot open file "%s".\n' "${repo_db}" + usage + fi + mkdir "${tmp_dir}/${repo}.${archive}" + bsdtar -C "${tmp_dir}/${repo}.${archive}" -xf "${repo_db}" + done +done + +# move the packages +for package in "$@"; do + errors=$( + find "${tmp_dir}/to.db" "${tmp_dir}/to.files" -mindepth 1 -maxdepth 1 \ + -printf '%f\n' | \ + sed 's/-[^-]\+-[^-]\+$//' | \ + grep -xF "${package%-*-*}" + ) + if [ -n "${errors}" ]; then + >&2 printf 'The target repository "%s" already contains the following packages - "repo-remove" them first:\n' \ + "${to_repo}" + >&2 printf '%s\n' "${errors}" + exit 2 + fi + for archive in 'db' 'files'; do + if [ ! -d "${tmp_dir}/from.${archive}/${package}" ]; then + >&2 printf 'Repository "%s" does not contain package "%s"\n' \ + "${from_repo}" "${package}" + exit 2 + fi + mv "${tmp_dir}/from.${archive}/${package}" "${tmp_dir}/to.${archive}/" + done +done + +# pack the database +for archive in 'db' 'files'; do + repo_db="${to_repo}" + if [ "${archive}" = 'files' ]; then + repo_db="${repo_db%.db.tar.gz}.files.tar.gz" + fi + bsdtar -C "${tmp_dir}" -czf "${tmp_dir}/${repo_db##*/}" --strip-components=1 "to.${archive}" +done + +# move the database in place +for archive in 'db' 'files'; do + repo_db="${to_repo}" + if [ "${archive}" = 'files' ]; then + repo_db="${repo_db%.db.tar.gz}.files.tar.gz" + fi + mv "${tmp_dir}/${repo_db##*/}" "${repo_db}" +done diff --git a/bin/sanity-check b/bin/sanity-check index 601a940..824b7a2 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -395,7 +395,14 @@ while [ $# -gt 0 ]; do if [ ${silence} -le 1 ]; then printf '\nThere is something wrong with the database:\n' cat "${webserver_directory}/mysql-sanity.html" - fi + fi >> \ + "${tmp_dir}/messages" + i_am_insane + fi + + if find "${work_dir}" -mindepth 1 -maxdepth 1 -name 'tmp.mysql-functions.query.*' | \ + grep '\S' >> \ + "${tmp_dir}/messages"; then i_am_insane fi diff --git a/lib/mysql-functions b/lib/mysql-functions index 26d3db8..d87b79e 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -25,31 +25,70 @@ base64_encode_each() { # mysql_run_query # wrapper function to query mysql mysql_run_query() { + local query_file if [ -s "${work_dir}/build-master-sanity" ]; then + # If the build master is insane, the calling command should only check + # if the build master is still insane - we do not want to log that. ${mysql_command} -N --raw --batch "$@" else - { - printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" - date - } \ - | tee -a "${work_dir}/mysql.stdin" \ - | tee -a "${work_dir}/mysql.stdout" \ - >> "${work_dir}/mysql.stderr" - tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} -N --raw --batch "$@" \ - 2>> "${work_dir}/mysql.stderr" \ - | tee -a "${work_dir}/mysql.stdout" - for s in \ - "${work_dir}/mysql.stdin" \ - "${work_dir}/mysql.stdout" \ - "${work_dir}/mysql.stderr"; do + # we save the query in a file and delete that file if the query succeeded + query_file=$(mktemp "${work_dir}/tmp.mysql-functions.query.$(date +'%Y-%m-%dT%T').XXXXXX") + cat > "${query_file}" + for i in {1..10}; do { - tail -n 10000 "$s" - printf '%s %s done: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date } \ - | sponge "$s" + | tee -a "${work_dir}/mysql.stdin" \ + | tee -a "${work_dir}/mysql.stdout" \ + >> "${work_dir}/mysql.stderr" + cat "${query_file}" \ + >> "${work_dir}/mysql.stdin" + { + ${mysql_command} -N --raw --batch "$@" \ + < "${query_file}" \ + 2>> "${work_dir}/mysql.stderr" \ + && rm "${query_file}" + } \ + | tee -a "${work_dir}/mysql.stdout" + if ! [ -f "${query_file}" ]; then + # success! + for s in \ + "${work_dir}/mysql.stdin" \ + "${work_dir}/mysql.stdout" \ + "${work_dir}/mysql.stderr"; do + { + tail -n 10000 "$s" + printf '%s %s done: ' "$0" "${mysql_command} $*" + date + } \ + | sponge "$s" + done + break + fi + for s in \ + "${work_dir}/mysql.stdin" \ + "${work_dir}/mysql.stdout" \ + "${work_dir}/mysql.stderr"; do + { + printf '%s %s FAILED: ' "$0" "${mysql_command} $*" + date + } \ + >> "$s" + done done + # a present query_file means there was an error + if [ -f "${query_file}" ]; then + >&2 printf 'I could not complete a mysql query!\n' + if [ ! -s "${work_dir}/build-master-sanity" ]; then + printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ + "${query_file##*/}" \ + | sponge "${irc_dir}/#archlinux-ports/in" + fi + echo 'A mysql query failed.' > \ + "${work_dir}/build-master-sanity" + return 2 + fi fi } |