#!/bin/sh
# do some basic sanity checks
# shellcheck disable=SC2119,SC2120
# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"
# TODO: remove hard-coded package suffixes
usage() {
>&2 echo ''
>&2 echo 'sanity-check [options] [checks]: check sanity of build master'
>&2 echo ''
>&2 echo 'possible options:'
>&2 echo ' -h|--help: Show this help and exit.'
>&2 echo ' -q|--quiet: Only print errors found.'
>&2 echo ' -r|--really-quiet: Do not print anything.'
>&2 echo ' -w|--wait: If necessary, wait for lock blocking.'
[ -z "$1" ] && exit 1 || exit "$1"
}
exit_code=0
i_am_insane() {
if [ ! -s "${work_dir}/build-master-sanity" ]; then
# shellcheck disable=SC2119
printf '\001ACTION goes insane.\001\n' | \
irc_say
fi
echo 'build master is insane' > \
"${work_dir}/build-master-sanity"
echo 'SANITY CHECK FAILED' | \
tee -a "${tmp_dir}/messages" >&2
exit_code=1
}
eval set -- "$(
getopt -o hqrw \
--long help \
--long quiet \
--long really-quiet \
--long wait \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
block_flag='-n'
silence=0
# shellcheck disable=SC2016
repos=$(
{
printf 'SELECT DISTINCT `repositories`.`name`'
printf ' FROM `repositories`'
printf ' WHERE `repositories`.`is_on_master_mirror`;\n'
} | \
mysql_run_query
)
# shellcheck disable=SC2016
archs=$(
{
printf 'SELECT DISTINCT `architectures`.`name`'
printf ' FROM `architectures`'
printf ' WHERE EXISTS ('
printf 'SELECT 1'
printf ' FROM `repositories`'
printf ' WHERE `repositories`.`architecture`=`architectures`.`id`'
printf ' AND `repositories`.`is_on_master_mirror`'
printf ');\n'
} | \
mysql_run_query
)
while true
do
case "$1" in
-h|--help)
usage 0
;;
-q|--quiet)
silence=1
;;
-r|--really-quiet)
silence=2
;;
-w|--wait)
block_flag=''
;;
--)
shift
break
;;
*)
>&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
exit 42
;;
esac
shift
done
exec 9> "${sanity_check_lock_file}"
if ! verbose_flock ${block_flag} 9; then
>&2 echo 'Sanity check skipped, cannot acquire lock.'
exit
fi
mysql_cleanup
finish() {
{
printf '%s\n' \
'' \
'' \
'result of archlinux32 build master'"'"'s sanity check' \
'' \
''
printf '%s
\n' "$(date)"
sed 's|$|
|' "${tmp_dir}/messages"
printf '%s\n' \
'' \
''
} > \
"${webserver_directory}/master-sanity.html"
rm -rf --one-file-system "${tmp_dir}"
}
tmp_dir=$(mktemp -d 'tmp.sanity-check.XXXXXXXXXX' --tmpdir)
touch "${tmp_dir}/messages"
trap 'finish' EXIT
if [ $# -eq 0 ]; then
set -- git-repositories build-list mysql ssh-keys master-mirror-availability repos package-database track-state
fi
while [ $# -gt 0 ]; do
case "$1" in
master-mirror-availability)
[ ${silence} -gt 0 ] || \
printf 'checking if master mirror can be reached ...' | \
tee -a "${tmp_dir}/messages" >&2
for host in $(
printf '%s\n' \
"${master_mirror_sftp_command}" \
"${master_mirror_rsync_directory}" | \
sed '
s,^.*@\([^:/@]\+\)\([:/].*\)\?$,\1,
t
d
' | \
sort -u
); do
if ! ping -c1 "${host}" >/dev/null 2>&1; then
if [ ${silence} -le 1 ]; then
printf '\nThe master mirror %s cannot be pinged.\n' \
"${host}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
# if the master mirror cannot be reached, further tests do not make any sense
set --
fi
done
if ! ${master_mirror_rsync_command} ${master_mirror_rsync_directory} >/dev/null 2>&1; then
if [ ${silence} -le 1 ]; then
printf '\nThe master mirror %s cannot be reached via rsync.\n' \
"${host}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
# if the master mirror cannot be reached, further tests do not make any sense
set --
fi
if ! ${master_mirror_sftp_command} /dev/null 2>&1; then
if [ ${silence} -le 1 ]; then
printf '\nThe master mirror %s cannot be reached via sftp.\n' \
"${host}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
# if the master mirror cannot be reached, further tests do not make any sense
set --
fi
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
;;
git-repositories)
[ ${silence} -gt 0 ] || \
printf 'checking git repositories ...' | \
tee -a "${tmp_dir}/messages" >&2
for repo in ${repo_names}; do
eval 'repo_path="${repo_paths__'"${repo}"'}"'
repo_revision=$(
# shellcheck disable=SC2016
{
printf 'SELECT `git_repositories`.`head`'
printf ' FROM `git_repositories`'
printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
"$(printf '%s' "${repo}" | base64 -w0)"
} | \
mysql_run_query
)
if ! obj_type=$(git -C "${repo_path}" cat-file -t "${repo_revision}" 2>/dev/null); then
if [ ${silence} -le 1 ]; then
printf '\nThe repository %s (%s) does not know the current revision %s.\n' \
"${repo}" "${repo_path}" "${repo_revision}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
elif [ "${obj_type}" != 'commit' ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe repository %s (%s) knows the current revision %s, but it is not a commit, but a %s.\n' \
"${repo}" "${repo_path}" "${repo_revision}" "${obj_type}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
done
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
;;
build-list)
[ ${silence} -gt 0 ] || \
printf 'checking build-list ...' | \
tee -a "${tmp_dir}/messages" >&2
errors=$(
# shellcheck disable=SC2016
{
printf 'SELECT `architectures`.`name`,`package_sources`.`pkgbase`'
printf ' FROM `package_sources`'
mysql_join_package_sources_build_assignments
mysql_join_build_assignments_architectures
printf ' WHERE EXISTS('
printf 'SELECT 1'
printf ' FROM `binary_packages`'
mysql_join_binary_packages_binary_packages_in_repositories
printf ' WHERE `binary_packages_in_repositories`.`repository`=%s' \
"${repository_ids__any_build_list}"
printf ' AND `binary_packages`.`build_assignment`=`build_assignments`.`id`'
printf ');\n'
} | \
mysql_run_query | \
sort | \
uniq -d
)
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe following packages have duplicate build orders:\n%s\n' \
"${errors}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
errors=$(
# shellcheck disable=SC2016
{
printf 'SELECT `a`.`pkgname`'
printf ' FROM `binary_packages` AS `a`'
mysql_join_binary_packages_binary_packages_in_repositories 'a' 'air'
mysql_join_binary_packages_in_repositories_repositories 'air' 'a_r'
printf ' AND `a_r`.`name`="build-list"'
printf ' JOIN `binary_packages` AS `b`'
printf ' ON `a`.`pkgname`=`b`.`pkgname`'
printf ' JOIN `architecture_compatibilities` AS `ac_a`'
printf ' ON `ac_a`.`fully_compatible`'
printf ' AND `ac_a`.`built_for`=`a`.`architecture`'
printf ' JOIN `architecture_compatibilities` AS `ac_b`'
printf ' ON `ac_b`.`fully_compatible`'
printf ' AND `ac_b`.`built_for`=`b`.`architecture`'
printf ' AND `ac_a`.`runs_on`=`ac_b`.`runs_on`'
mysql_join_binary_packages_binary_packages_in_repositories 'b' 'bir'
mysql_join_binary_packages_in_repositories_repositories 'bir' 'b_r'
printf ' AND `b_r`.`name`="deletion-list";\n'
} | \
mysql_run_query
)
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe following packages appear on the build- and deletion-list:\n%s\n' \
"${errors}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
;;
repos)
[ ${silence} -gt 0 ] || \
printf 'checking repos on master mirror ...' | \
tee -a "${tmp_dir}/messages" >&2
errors=$(
{
# shellcheck disable=SC2086
for arch in ${archs}; do
printf 'expected '"${arch}"' %s\n' ${repos}
ls_master_mirror "${arch}" | \
sed 's|^|found '"${arch}"' |'
done
} | \
sort -k2,3 | \
uniq -uf1
)
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe following repos are missing or obsolete on the mirror:\n%s\n' \
"${errors}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
;;
package-database)
for arch in ${archs}; do
for repo in ${repos}; do
[ ${silence} -gt 0 ] || \
printf 'checking consistency of repository "%s/%s" on the master mirror ...' "${arch}" "${repo}" | \
tee -a "${tmp_dir}/messages" >&2
packages=$(
ls_master_mirror "${arch}/${repo}" | \
grep '\.pkg\.tar\.xz\(\.sig\)\?$'
) || true
errors=$(
echo "${packages}" | \
grep '\S' | \
sed '
s|^\(.*\.pkg\.tar\.xz\)$|package \1|
s|^\(.*\.pkg\.tar\.xz\)\.sig$|signature \1|
' | \
sort -k2 | \
uniq -cf1 | \
grep -v '^\s*2\s' | \
awk '{print $2 " " $3}'
) || true
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe following packages in %s are missing a signature or vice versa:\n%s\n' \
"${repo}" \
"${errors}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
failsafe_rsync \
"${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db.tar.gz" \
"${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files.tar.gz" \
"${tmp_dir}/"
errors=$(
{
tar -Oxzf "${tmp_dir}/${repo}.db.tar.gz" --wildcards '*/desc' 2>/dev/null | \
sed -n '
/^%FILENAME%$/ {
N
s/^.*\n/in_database /
p
}
'
echo "${packages}" | \
sed '
/\.pkg\.tar\.xz$/ !d
s/^/in_repository /
' | \
sort -u
} | \
sort -k2 | \
uniq -uf1
)
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe following packages in %s are missing from the database or vice versa:\n%s\n' \
"${repo}" \
"${errors}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
errors=$(
{
tar -tzf "${tmp_dir}/${repo}.files.tar.gz" 2>/dev/null | \
grep '/$' | \
sed '
s|/$||
s|^|in_database |
'
echo "${packages}" | \
grep '\S' | \
sed '
s|-[^-]\+$||
s|^|in_repository |
' | \
sort -u
} | \
sort -k2 | \
uniq -uf1
)
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe following packages in %s are missing from the file-database or vice versa:\n%s\n' \
"${repo}" \
"${errors}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
find "${tmp_dir:?}" -mindepth 1 \( -not -name 'messages' \) -delete
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
done
done
;;
track-state)
[ ${silence} -gt 0 ] || \
printf 'checking if all packages are tracked correctly ...' | \
tee -a "${tmp_dir}/messages" >&2
errors=$(
{
# shellcheck disable=SC2016
{
printf 'SELECT "mysql",CONCAT(`r_a`.`name`,"/",`repositories`.`name`,"/",'
mysql_package_name_query
printf ') FROM `binary_packages`'
mysql_join_binary_packages_binary_packages_in_repositories
mysql_join_binary_packages_in_repositories_repositories
printf ' AND `repositories`.`is_on_master_mirror`'
mysql_join_binary_packages_architectures
mysql_join_repositories_architectures '' 'r_a'
} | \
mysql_run_query | \
tr '\t' ' '
for arch in ${archs}; do
ls_master_mirror "${arch}" | \
while read -r repo; do
ls_master_mirror "${arch}/${repo}" | \
sed '
/\.pkg\.tar\.xz$/!d
s,^,package-file '"${arch}"'/'"${repo}"'/,
'
done
done
} | \
sort -k2 | \
uniq -uf1
)
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe following packages from the master mirror are not tracked in the database or vice versa:\n%s\n' \
"${errors}" | \
tee -a "${tmp_dir}/messages" >&2
fi
i_am_insane
fi
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
;;
mysql)
[ ${silence} -gt 0 ] || \
printf 'checking mysql-sanity ...' | \
tee -a "${tmp_dir}/messages" >&2
mysql_sanity_check | \
sed '
s,^-.*$,\0,
s,^+.*$,\0,
s/$/
/
1 i sanity of the buildmaster'"'"'s mysql database
$ a
' | \
sponge "${webserver_directory}/mysql-sanity.html"
if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
[ ! -s "${work_dir}/build-master-sanity" ]; then
# shellcheck disable=SC2119
{
printf 'girls, my database is dirty again ...\n'
printf 'dirty! girls, my database - so dirty :-(\n'
printf 'girls, please have a look at my dirty database'
} | \
shuf -n1 | \
irc_say
fi
if [ -s "${webserver_directory}/mysql-sanity.html" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThere is something wrong with the database:\n'
cat "${webserver_directory}/mysql-sanity.html"
fi | \
tee -a "${tmp_dir}/messages" >&2
i_am_insane
fi
# hopefully, this gets rid of false positives :-)
sleep 1
errors=$(
find "${work_dir}" -mindepth 1 -maxdepth 1 \
-not -name 'tmp.mysql-functions.unimportant_query.*' \
-name 'tmp.mysql-functions.query.*' | \
parallel -j100 \
'bash -c "
sleep 5
test -s {} && '\
'printf '"'"'%s:\n'"'"' {} && '\
'sed '"'"'s/^/>> /'"'"' {}
"'
) || \
true
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThere are pending mysql queries:\n%s\n' \
"${errors}"
fi | \
tee -a "${tmp_dir}/messages" >&2
i_am_insane
fi
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
;;
ssh-keys)
[ ${silence} -gt 0 ] || \
printf 'checking ssh-keys ...' | \
tee -a "${tmp_dir}/messages" >&2
# shellcheck disable=SC2016
{
printf 'SELECT'
printf ' CONCAT('
printf '"command=\\\"%s/bin/slave-build-connect ",' \
"${base_dir}"
printf '`build_slaves`.`name`,'
printf '"\\\" ssh-rsa ",'
printf '`ssh_keys`.`fingerprint`'
printf ')'
printf ' FROM'
printf ' `build_slaves`'
mysql_join_build_slaves_ssh_keys
printf ' WHERE `build_slaves`.`access_allowed`'
} | \
mysql_run_query | \
sort > \
"${tmp_dir}/ssh-keys.mysql"
sed -n '
/^command/ {
s/^\(\(\S\+\s\+\)\{4\}\)\S.*$/\1/
s/\s\+$//
p
}
' ~/".ssh/authorized_keys" | \
sort > \
"${tmp_dir}/ssh-keys.authorized_keys"
errors=$(
diff "${tmp_dir}/ssh-keys.mysql" "${tmp_dir}/ssh-keys.authorized_keys"
) || true
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
printf '\nThe ssh keys in the database and the file system differ:\n%s\n' \
"${errors}"
fi | \
tee -a "${tmp_dir}/messages" >&2
i_am_insane
fi
[ ${silence} -gt 0 ] || \
echo ' passed.' | \
tee -a "${tmp_dir}/messages" >&2
;;
*)
[ ${silence} -gt 1 ] || \
>&2 printf 'unknown sanity-check "%s".\n' "$1"
exit 2
;;
esac
shift
done
if [ ${exit_code} -ne 0 ]; then
exit ${exit_code}
fi
if [ -f "${work_dir}/build-master-sanity" ]; then
rm "${work_dir}/build-master-sanity"
# shellcheck disable=SC2119
printf '\001ACTION resumes sanity.\001\n' | \
irc_say
fi