From 2e6d97b8ddcd16f029673bc6f101fc3baf141e51 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sat, 24 Feb 2018 19:41:56 +0100 Subject: bin/common-functions -> lib/common-functions, bin/mysql-functions -> lib/mysql-functions --- lib/common-functions | 1378 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1378 insertions(+) create mode 100755 lib/common-functions (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions new file mode 100755 index 0000000..a84271b --- /dev/null +++ b/lib/common-functions @@ -0,0 +1,1378 @@ +#!/bin/sh + +# contains functions used by more than one script + +# shellcheck disable=SC2039 + +# TODO: include link depenendencies in run-depends metadata + +# TODO: have full information (currently in files) in database + +# TODO: remove state files / metadata files + +if [ -z "${base_dir}" ]; then + # just to make shellcheck happy + . 'conf/default.conf' +fi + +# find_pkgbuilds package repository git_repository git_revision mod_git_revision +# find the PKGBUILD and modification of $package from $repository +# sets $PKGBUILD and $PKGBUILD_mod + +find_pkgbuilds() { + + local package="$1" + local repository="$2" + local git_repository="$3" + local git_revision="$4" + local mod_git_revision="$5" + + local repo_path + eval 'repo_path="${repo_paths__'"${git_repository}"'}"' + + PKGBUILD=$( + git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \ + tar -t 2> /dev/null | \ + grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \ + grep -v -- '-i686/PKGBUILD$' | \ + grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ + sort | \ + tail -n1 + ) + + PKGBUILD_mod=$( + git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \ + tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null + ) || true + + if [ -z "${PKGBUILD}" ] && \ + [ -z "${PKGBUILD_mod}" ]; then + >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \ + "${package}" \ + "${repository}" \ + "${git_repository}" \ + "${git_revision}" \ + "${mod_git_revision}" + return 1 + fi + +} + +# find_repository_with_commit commit +# find the repository which has $commit + +find_repository_with_commit() { + + local repository + + for repository in ${repo_names}; do + # shellcheck disable=SC2016 + if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then + echo "${repository}" + return 0 + fi + done + >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1" + exit 1 + +} + +# find_git_repository_to_package_repository repository +# find the git repository which tracks the package repository $repository + +find_git_repository_to_package_repository() { + + local repository + local package_repository + local repo_path + + package_repository="$1" + + if [ "$1" = 'build-support' ]; then + echo 'packages' + return 0 + fi + + for repository in ${repo_names}; do + if [ "${repository}" = "archlinux32" ]; then + continue + fi + eval 'repo_path="${repo_paths__'"${repository}"'}"' + if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \ + tar -t --wildcards '*/repos' | \ + grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \ + cut -d/ -f3 | \ + sed 's|-[^-]\+$||' | \ + sort -u | \ + grep -qxF "${package_repository}"; then + echo "${repository}" + return 0 + fi + done + >&2 echo "can't find git repository with package repository '$1'" + exit 1 + +} + +# generate_package_metadata $package $git_revision $mod_git_revision $repository +# or +# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository +# generate the meta data files of a package (dependencies, built packages, ...) + +generate_package_metadata() { + + local package="$1" + local git_revision="$2" + local mod_git_revision="$3" + local repository="$4" + local file_prefix + local file + local PKGBUILD + + if [ $# -eq 1 ]; then + # second form + repository="${package##*.}" + package="${package%.*}" + mod_git_revision="${package##*.}" + package="${package%.*}" + git_revision="${package##*.}" + package="${package%.*}" + fi + + file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}" + + if [ -e "${file_prefix}.builds" ] && \ + [ -e "${file_prefix}.build-depends" ] && \ + [ -e "${file_prefix}.run-depends" ] && \ + [ -e "${file_prefix}.groups" ] && \ + [ -e "${file_prefix}.packages" ]; then + return 0 + fi + + if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then + printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO" + exit 1 + fi + if [ ! -s "${file_prefix}.SRCINFO" ]; then + >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO" + exit 1 + fi + + # otherwise this just calls for trouble + sed -i ' + /^[^=]*=\s*$/d + s/_i686\(\s*=\)/\1/ + ' "${file_prefix}.SRCINFO" + + # extract "groups" = groups \cup provides + grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u > \ + "${file_prefix}.groups" + + # extract "packages" = pkgname + grep '^pkgname = ' "${file_prefix}.SRCINFO" | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u > \ + "${file_prefix}.packages" + + # extract "builds" = provides \cup pkgname \cup groups + cat "${file_prefix}.groups" "${file_prefix}.packages" | \ + sort -u > \ + "${file_prefix}.builds" + + # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds" + { + { + printf 'all_depend = %s\n' 'base' 'base-devel' + sed -n "$( + printf '/^\t%s = /p\n' \ + 'depends' \ + 'makedepends' + )" "${file_prefix}.SRCINFO" + } | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u + sed 'p' "${file_prefix}.builds" + } | \ + sort | \ + uniq -u > \ + "${file_prefix}.build-depends" + + # extract "run-depends" = depends \cup \{ base \} \setminus "builds" + { + { + printf 'all_depend = %s\n' 'base' + sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO" + } | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u + sed 'p' "${file_prefix}.builds" + } | \ + sort | \ + uniq -u > \ + "${file_prefix}.run-depends" + + rm "${file_prefix}.SRCINFO" + +} + +# delete_old_metadata +# delete old (=unneeded) meta data of packages + +delete_old_metadata() { + + local current_metadata + + current_metadata=$( + find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ + sed ' + s|\.[^.]\+$|| + s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3| + ' | \ + sort -u + ) + + ( # the new shell is intentional + # what we have + echo "${current_metadata}" + + # package-states should stay + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ + sed ' + s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| + ' | \ + sort -u | \ + sed 'p' + + # build-list items should stay + sed 'p' "${work_dir}/build-list" + + tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + echo "${current_metadata}" | \ + sort -k1,1 > \ + "${tmp_dir}/current-metadata" + + # the newest of the following should stay: + { + # deletion-list items + cat "${work_dir}/deletion-list" + # all packages in the repos + for repo in ${repo_names}; do + eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \ + tar -t | \ + sed ' + s|/$|| + /\//d + ' + done + } | \ + sort -u | \ + join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \ + sort -k4,4 > \ + "${tmp_dir}/find-newest-revisions" + + uniq -uf3 < \ + "${tmp_dir}/find-newest-revisions" | \ + awk '{print $4 " " $1 " " $2 " " $3}' | \ + sed 'p' + + uniq -Df3 < \ + "${tmp_dir}/find-newest-revisions" | \ + uniq --group=append -f3 | \ + { + revs='' + mod_revs='' + opkg='' + orepo='' + while read -r rev mod_rev repo pkg; do + + if [ -z "${rev}" ] && \ + [ -z "${mod_rev}" ] && \ + [ -z "${repo}" ] && \ + [ -z "${pkg}" ]; then + + printf '%s %s %s %s\n' \ + "$( + printf '%s\n' ${revs} | \ + find_newest_of_git_revisions + )" \ + "$( + printf '%s\n' ${mod_revs} | \ + find_newest_of_git_revisions + )" \ + "${orepo}" \ + "${opkg}" + + revs='' + mod_revs='' + orepo='' + opkg='' + continue + fi + revs=$( + # shellcheck disable=SC2086 + printf '%s\n' ${revs} ${rev} | \ + sort -u + ) + mod_revs=$( + # shellcheck disable=SC2086 + printf '%s\n' ${mod_revs} ${mod_rev} | \ + sort -u + ) + orepo="${repo}" + opkg="${pkg}" + done + } | \ + awk '{print $4 " " $1 " " $2 " " $3}' | \ + sed 'p' + ) | \ + sort | \ + uniq -u | \ + while read -r pkg rev mod_rev repo; do + rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."* + done +} + +# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository +# print which (stable) repository a package belongs to + +repository_of_package() { + local package="$1" + local repository="${package##*.}" + package="${package%.*}" + local a32_rev="${package##*.}" + package="${package%.*.*}" + + case "${repository}" in + 'multilib') + if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \ + tar -Ox | \ + grep -qFx "${package%.*.*.*}"; then + echo 'extra' + else + echo 'community' + fi + ;; + *) + echo "${repository}" + esac +} + +# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending +# print wether the specified package is an official package (print +# $ending) or a community package (print 'community-$ending') or a +# build-support package (print 'build-support') + +official_or_community() { + local prepo + prepo=$(repository_of_package "$1") + + if [ "${prepo}" = 'community' ]; then + echo 'community-'"$2" + elif [ "${prepo}" = 'build-support' ]; then + echo 'build-support' + else + echo "$2" + fi +} + +# ls_master_mirror $path +# list content of $path on the master mirror (via rsync) + +ls_master_mirror() { + + local path="$1" + + ${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/${path}/" | \ + grep -v '\s\.$' | \ + awk '{print $5}' + +} + +# TODO: the actions of remove_old_package_versions should be done +# on basis of the information in the database + +# remove_old_package_versions $arch $repository $package_file + +# removes all older (not-newer) versions of $package_file +# in all repositories not-older (newer) than $repository + +# TODO: should remove all other version (also newer) from +# some repositories :-/ + +# A package is considered not newer if +# a) its version is not newer +# A package is considered older if +# b) its version is older or +# c) if it's "not newer" and its architecture is 'any' and different or +# d) if it's "not newer" and the other architecture is 'any' and different + +# this ensures an any package may replace arch-specific packages of the same version and vice versa + +remove_old_package_versions() { + + local arch="$1" + local repository="$2" + local package="$3" + + local pkgname + local epoch + local pkgver + local pkgrel + local sub_pkgrel + pkgname="${package%-*}" + pkgrel="${pkgname##*-}" + sub_pkgrel="${pkgrel##*.}" + if [ "${sub_pkgrel}" = "${pkgrel}" ]; then + sub_pkgrel='0' + else + pkgrel="${pkgrel%.*}" + fi + pkgname="${pkgname%-*}" + pkgver="${pkgname##*-}" + epoch="${pkgver%%:*}" + if [ "${epoch}" = "${pkgver}" ]; then + epoch='0' + else + pkgver="${pkgver#*:}" + fi + pkgname="${pkgname%-*}" + + # shellcheck disable=SC2016 + { + printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \ + "$( + printf '%s' "${repository}" | \ + base64 -w0 + )" \ + "$( + printf '%s' "${package}" | \ + base64 -w0 + )" \ + "$( + printf '%s' "${package}" | \ + sed ' + s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/ + ' | \ + base64 -w0 + )" + printf 'SELECT ' + printf '`binary_packages`.`id`,' + printf 'CONCAT(`repositories`.`name`,"/",' + mysql_package_name_query + printf '),' + # should we delete packages of identical version? + printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),' + printf 'CONCAT(' + printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),' + printf '`binary_packages`.`pkgver`,"-",' + printf '`binary_packages`.`pkgrel`,".",' + printf '`binary_packages`.`sub_pkgrel`' + printf ')' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`' + printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`' + # name must match + printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \ + "$(printf '%s' "${package%-*-*-*}" | base64 -w0)" + # repository, where package should be deleted, should be less stable + printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \ + "$(printf '%s' "${repository}" | base64 -w0)" + printf ';\n' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + /^\S\+\sCONCAT(/d + y/\t/ / + ' | \ + expand_version 4 | \ + sort -k4V,4 -k3r,3 | \ + shrink_version 4 | \ + sed -n ' + /^bogus /q + p + ' | \ + cut -d' ' -f1,2 >&2 + + # repositories in which older packages should be deleted + local delete_older_repositories + # repositories in which not-newer packages should be deleted + local delete_not_newer_repositories + + if echo "${standalone_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories="${repository}" + delete_not_newer_repositories='' + + elif echo "${staging_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories="${repository}" + delete_not_newer_repositories=$( + echo "${staging_package_repositories}" | \ + grep -vxF "${repository}" + ) || true + + elif echo "${testing_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories=$( + printf '%s\n' "${staging_package_repositories}" "${repository}" + ) + delete_not_newer_repositories=$( + echo "${testing_package_repositories}" | \ + grep -vxF "${repository}" + ) || true + + elif echo "${stable_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories=$( + printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}" + ) + delete_not_newer_repositories=$( + echo "${stable_package_repositories}" | \ + grep -vxF "${repository}" + ) || true + + else + + >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}" + return 1 + + fi + + ( # the new shell is intentional + tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + { + # the architecture of the package (any vs. i686) + package_arch="${package##*-}" + package_arch="${package_arch%%.*}" + if [ "${package_arch}" = 'any' ]; then + package_arch_regex_inverter='!' + else + unset package_arch_regex_inverter + fi + + for repo in ${delete_older_repositories}; do + ls_master_mirror "${arch}/${repo}" | \ + sed -n ' + /\.pkg\.tar\.xz$/!d + s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| + /^'"$(str_to_regex "${package%-*-*-*}")"' / { + s|^|2 '"${arch} ${repo}"' | + / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{ + s|^2|0| + } + p + } + ' + done + for repo in ${delete_not_newer_repositories}; do + ls_master_mirror "${arch}/${repo}" | \ + sed -n ' + /\.pkg\.tar\.xz$/!d + s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| + /^'"$(str_to_regex "${package%-*-*-*}")"' / { + s|^|0 '"${arch} ${repo}"' | + p + } + ' + done + echo "${package%-*}" | \ + sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|' + + # the generated list contains the following columns: + # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz + } | \ + expand_version 5 | \ + sort -k5V,5 -k1n,1 | \ + shrink_version 5 | \ + sed -n ' + /^1 %cut% %it% %here% /q + s/^[02] // + s/ \(\S\+\)$/-\1/ + p + ' | \ + sort -u > \ + "${tmp_dir}/packages-to-delete" + # this file contains a list of packages to be deleted, one on each line: + # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz + + cut -d' ' -f1,2 < \ + "${tmp_dir}/packages-to-delete" | \ + grep -vxF "${arch} ${repository}" | \ + sort -u > \ + "${tmp_dir}/repositories-to-modify" + + # fetch all databases being modified + while read -r del_arch del_repo; do + mkdir -p "${tmp_dir}/${del_arch}/${del_repo}" + ${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \ + "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \ + "${tmp_dir}/${del_arch}/${del_repo}/" + done < \ + "${tmp_dir}/repositories-to-modify" + + while read -r del_arch del_repo del_package _; do + if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then + # we do not repo-remove the package in the target repository + continue + fi + repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \ + "${del_package}" + done < \ + "${tmp_dir}/packages-to-delete" + + # upload modified databases + while read -r del_arch del_repo; do + ${master_mirror_rsync_command} \ + "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \ + "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \ + "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/" + done < \ + "${tmp_dir}/repositories-to-modify" + + # shellcheck disable=SC2016 + sed ' + s/\.pkg\.tar\.xz$// + s/^\S\+ // + s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/ + s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/ + s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/ + ' "${tmp_dir}/packages-to-delete" | \ + while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do + printf 'DELETE FROM `binary_packages` WHERE' + printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \ + 'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \ + 'repository' 'repositories' 'repositories' 'repositories' "$(printf '%s' "${repo}" | base64 -w0)" + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \ + 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ + 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ + 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ + 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \ + sed 's/ AND$//' + printf ';\n' + done | \ + ${mysql_command} + + sed ' + s| \(\S\+\)$|-\1| + y| |/| + s|^|rm "| + s|$|"| + p + s|"$|.sig"| + ' "${tmp_dir}/packages-to-delete" | \ + ${master_mirror_sftp_command} + ) + +} + +# wait_some_time $minimum $diff +# wait between minimum and minimum+diff seconds (diff defaults to 30) + +wait_some_time() { + local minimum=$1 + local diff=$2 + local random + + if [ -z "${diff}" ]; then + diff=30 + fi + + random=$( + dd if='/dev/urandom' count=1 2> /dev/null | \ + cksum | \ + cut -d' ' -f1 + ) + + sleep $((minimum + random % diff)) +} + +# str_to_regex $string +# escape dots for use in regex + +str_to_regex() { + echo "$1" | \ + sed ' + s|[.[]|\\\0|g + ' +} + +# make_source_info $package $repository $git_revision $mod_git_revision $output +# create .SRCINFO from PKGBUILD within git repositories, output to $output + +make_source_info() { + + local package="$1" + local repository="$2" + local git_revision="$3" + local mod_git_revision="$4" + local output="$5" + + local git_repo + local PKGBUILD + local PKGBUILD_mod + + git_repo=$(find_repository_with_commit "${git_revision}") + + if [ -z "${git_repo}" ]; then + return 1 + fi + + find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" + + ( # the new shell is intentional + + tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX") + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0' + + { + cd "${tmp_dir}" + makepkg --printsrcinfo + cd .. + } | + if [ "${package%-i18n}-i18n" = "${package}" ]; then + sed ' + 1 a \\tdepends = '"${package%-i18n}"' + ' + else + cat + fi > \ + "${output}" + + ) + +} + +# recursively_umount_and_rm $dir +# umount all mountpoints in $dir which are also in $dir's +# filesystem, possibly also $dir itself and then +# rm -rf --one-file-system $dir + +recursively_umount_and_rm() { + local dir="$1" + + if [ -z "${dir}" ]; then + >&2 echo 'ERROR: recursively_umount_and_rm requires an argument' + exit 42 + fi + + find "${dir}" \ + -xdev -depth -type d \ + -exec 'mountpoint' '-q' '{}' ';' \ + -exec 'sudo' 'umount' '-l' '{}' ';' + rm -rf --one-file-system "${dir}" +} + +# mangle_pkgbuild $PKGBUILD [$sub_pkgrel] +# mangle $arch in PKBUILDs to contain i486, i586, i686 +# append $sub_pkgrel to the pkgrel + +mangle_pkgbuild() { + local PKGBUILD="$1" + local sub_pkgrel="$2" + + if [ -n "${sub_pkgrel}" ]; then + sub_pkgrel=".${sub_pkgrel}" + fi + + sed -i ' + /^arch=[^#]*any/!{ + /^arch=(/s/(/(i686 / + } + s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ + ' "${PKGBUILD}" +} + +# find_newest_of_git_revisions +# find newest git revision of the ones provided at stdin +# (assuming linear history) + +find_newest_of_git_revisions() { + local revisions + local repo + revisions=$(cat) + + if [ "$( + echo "${revisions}" | \ + wc -l + )" -eq 1 ]; then + + echo "${revisions}" + return + + fi + + repo=$( + find_repository_with_commit \ + "$( + echo "${revisions}" | \ + grep -xm1 '[0-9a-f]\{40\}' + )" + ) + + eval 'repo="${repo_paths__'"${repo}"'}"' + + echo "${revisions}" | \ + xargs -rn1 git -C "${repo}" rev-parse | \ + { + newest='' + while read -r current; do + if [ -z "${newest}" ] || \ + git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then + newest="${current}" + fi + done + echo "${newest}" + } +} + +# find_package_repository_to_package $package $git_repository $git_commit +# find the package repository a package from a given git repository +# belongs to + +find_package_repository_to_package() { + + local package="$1" + local git_repository="$2" + local git_commit="$3" + local repo_path + local repo + + eval 'repo_path="${repo_paths__'"${git_repository}"'}"' + + if [ "${git_repository}" = 'archlinux32' ]; then + repo=$( + git -C "${repo_path}" archive "${git_commit}" -- | \ + tar -t --wildcards "*/${package}/" | \ + cut -d/ -f1 | \ + sort -u + ) + else + repo=$( + git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \ + tar -t | \ + cut -d/ -f3 | \ + grep -vxF '' | \ + grep -v 'staging\|testing\|-unstable' | \ + grep -v -- '-i686$' | \ + sed 's|-[^-]\+$||' | \ + sort -u + ) + fi + + if [ -z "${repo}" ]; then + return 1 + fi + + if [ "$( + echo "${repo}" | \ + wc -l + )" -ne 1 ]; then + return 1 + fi + + echo "${repo}" + +} + +# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel +# extract files found in the svn/git source directories +# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly + +extract_source_directory() { + + local git_repo="$1" + local rev="$2" + local mod_rev="$3" + local output="$4" + local sub_pkgrel="$5" + + if [ -n "${PKGBUILD}" ]; then + eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \ + tar -x --strip-components=3 -C "${output}" + fi + + if [ -n "${PKGBUILD_mod}" ]; then + git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \ + tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \ + true + git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \ + tar -Ox "${PKGBUILD_mod}" >> \ + "${output}/PKGBUILD" + fi + + mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}" + + # shellcheck disable=SC2016 + sed -i '/^\$Id\$$/d' "${output}/PKGBUILD" + +} + +# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository +# return a list of packages on the build list which are (run- / build- / check-time) +# dependencies of the given package + +find_dependencies_on_build_list() { + + local package="$1" + local git_revision="$2" + local mod_git_revision="$3" + local repository="$4" + + generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" + + { + cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" + awk '{print $1 "." $2 "." $3 "." $4}' < \ + "${work_dir}/build-list" | \ + sed ' + s|^|'"${work_dir}"'/package-infos/| + s|$|\.builds| + ' | \ + xargs -r cat | \ + sort -u + } | \ + sort | \ + uniq -d + +} + +# download_sources_by_hash $package $repository $git_revision $git_mod_revision +# try to download all sources by their hash into the current directory +# returns 0 if any source was downloaded and 1 otherwise + +download_sources_by_hash() { + + local package="$1" + local repository="$2" + local git_revision="$3" + local git_mod_revision="$4" + + local return_value=1 + local tmp_dir + local sum_type + local arch_suffix + + tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir) + + if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then + >&2 echo 'download_sources_by_hash: make_source_info failed.' + rm -rf --one-file-system "${tmp_dir}" + return 1 + fi + + if ! [ -s "${tmp_dir}/.SRCINFO" ]; then + >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.' + rm -rf --one-file-system "${tmp_dir}" + return 1 + fi + + for arch_suffix in '' '_i686'; do + for sum_type in 'sha256sum' 'sha512sum'; do + grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ + sed 's|^.* = ||' | \ + cat -n > \ + "${tmp_dir}/sums" + grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ + sed ' + s|^.* = || + s|::.*$|| + s|.*/|| + ' | \ + cat -n > \ + "${tmp_dir}/urls" + if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then + join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \ + "${tmp_dir}/joined" + while read -r sum file; do + if [ "${sum}" = 'SKIP' ]; then + continue + fi + if echo "${sum} ${file}" | \ + ${sum_type} -c > /dev/null 2>&1; then + # the correct source is already there + continue + fi + if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then + mv "${tmp_dir}/transfer" "${file}" + return_value=0 + fi + done < \ + "${tmp_dir}/joined" + fi + done + done + + rm -rf --one-file-system "${tmp_dir}" + return ${return_value} + +} + +# expand_version $column_num +# add "0:" to version in $colum_num-th column if no ":" is there (epoch) +# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash) + +expand_version() { + local column_num + column_num="$1" + + sed ' + /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/ + /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/ + ' +} + +# shrink_version $column_num +# remove "0:" from version in $colum_num-th column (epoch) +# remove "+0" from version in $colum_num-th column (git count/hash) + +shrink_version() { + local column_num + column_num="$1" + + sed ' + s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/ + s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/ + ' +} + +# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ] + +# Return (to stdout) the biggest subset A of the packages in $omega whose +# runtime dependencies in $omega \cup $keep are also in A + +# $all_builds and $all_depends either point to an empty file - then they will get +# filled with cached data for subsequent calls - or to the same files of a previous +# call + +# If non-empty, $force contains packages which are assumed to match the above +# condition without checking. + +# The arguments are names of files with one $package.$revision.$mod_revision.$repository +# per line. + +find_biggest_subset_of_packages() { + + ( # the new shell is intentional + omega="$1" + keep="$2" + all_builds="$3" + all_depends="$4" + if [ $# -eq 4 ]; then + force='/dev/null' + elif [ $# -eq 5 ]; then + force="$5" + else + >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#" + return 2 + fi + + if [ ! -s "${all_builds}" ]; then + find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \ + -exec sed ' + s|^|{} | + s|^\S\+/|| + s|\.builds | | + ' {} \; | \ + sort -k2,2 > \ + "${all_builds}" + fi + + if [ ! -s "${all_depends}" ]; then + find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \ + -exec sed ' + s|^|{} | + s|^\S\+/|| + s|\.run-depends | | + ' {} \; | \ + grep -v ' base$' | \ + sort -k2,2 > \ + "${all_depends}" + fi + + sort -u "${omega}" | \ + sponge "${omega}" + + temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + + { + sort -u "${keep}" + cat "${force}" "${force}" + } | \ + sort | \ + uniq -u > \ + "${temp_dir}/keep.new" + touch "${temp_dir}/keep" + + while [ -s "${temp_dir}/keep.new" ]; do + cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \ + sort -u | \ + sponge "${temp_dir}/keep" + + { + # we append all packages which are run-dependencies of keep-packages + # to the keep-list + sed ' + s|^|'"${work_dir}"'/package-infos/| + s|$|.run-depends| + ' "${temp_dir}/keep" | \ + xargs -r grep -HF '' | \ + sed ' + s|^.*/|| + s|\.run-depends:| | + ' | \ + sort -u | \ + sort -k2,2 | \ + uniq -f1 | \ + join -1 2 -2 2 -o 2.1 - "${all_builds}" + + # we append all packages with run-dependencies on the keep-list + # to the keep-list + sed ' + s|^|'"${work_dir}"'/package-infos/| + s|$|.builds| + ' "${temp_dir}/keep" | \ + xargs -r grep -HF '' | \ + sed ' + s|^.*/|| + s|\.builds:| | + ' | \ + sort -u | \ + sort -k2,2 | \ + uniq -f1 | \ + join -1 2 -2 2 -o 2.1 - "${all_depends}" + } | \ + sort -u | \ + join -1 1 -2 1 -o 2.1 - "${omega}" | \ + sort -u > \ + "${temp_dir}/keep.new" + + # "new" is only what has not been there before and what is not forced + cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \ + sort | \ + uniq -u | \ + sponge "${temp_dir}/keep.new" + done + + cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \ + sort | \ + uniq -u + + ) + +} + +# sort_square_bracket_content $file +# sort the content of [] in $file, print to stdout + +sort_square_bracket_content() { + local file + local line + local token + local token_list + local rest + file="$1" + + while read -r line; do + printf '%s ' "${line}" | \ + tr ' ' '\n' | \ + while read -r token; do + if echo "${token}" | \ + grep -qF '['; then + printf '%s[' "${token%[*}" + token="${token##*[}" + token_list="${token%,}" + while ! echo "${token_list}" | \ + grep -qF ']'; do + read -r token + token_list=$( + printf '%s\n' \ + "${token_list}" \ + "${token%,}" + ) + done + rest="]${token_list#*]}" + token_list="${token_list%%]*}" + token=$( + printf '%s' "${token_list}" | \ + sort | \ + sed 's|$|,|' + printf '%s' "${rest}" + ) + fi + printf '%s\n' "${token}" + done | \ + tr '\n' ' ' | \ + sed ' + s|, ]|]|g + s| $|| + ' + printf '\n' + done < \ + "${file}" +} + +# smoothen_namcap_log $file +# remove unneccesary differences from namcap-logs: +# - remove architecture specific information +# - sort lines +# - sort content of square brackets + +smoothen_namcap_log() { + local file + file="$1" + # shellcheck disable=SC2016 + sort_square_bracket_content "${file}" | \ + sed ' + # normalize architecture specific information + s|i[34567]86|$ARCH|g + s|x86\([-_]64\)\?|$ARCH|g + # remove haskell hashes + s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g + ' | \ + sort | \ + sponge "${file}" +} + +# print_list_of_archaic_packages $source1 $source2 ... +# print a list of packages which have not been touched for a while, +# but which are still in the pipeline, e.g. in $source1, $source2 or ... + +print_list_of_archaic_packages() { + for source in "$@"; do + case "${source}" in + 'testing') + # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32 + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \ + -exec head -n1 {} \; | \ + "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin + # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated + # and no longer block other packages from being moved + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \ + sed ' + s|\.testing$|| + ' + ;; + 'build-list') + while read -r pkg rev mod_rev repo; do + git_repo=$( + find_repository_with_commit "${rev}" + ) + eval repo_path='"${repo_paths__'"${git_repo}"'}"' + commit_date=$( + git -C "${repo_path}" show -s --format=%ct "${rev}" + ) + mod_commit_date=$( + git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}" + ) + if [ "${mod_commit_date}" -gt "${commit_date}" ]; then + commit_date="${mod_commit_date}" + fi + # packages remaining longer than $max_package_age_build_list days on the build list + if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then + printf '%s %s %s %s\n' \ + "${pkg}" \ + "${rev}" \ + "${mod_rev}" \ + "${repo}" + fi + done < \ + "${work_dir}/build-list" + ;; + 'staging') + # packages remaining longer than $max_package_age_staging days in staging + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \ + sed ' + s|\.done$|| + ' + ;; + *) + >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}" + ;; + esac + done | \ + sort -u +} + +# modification_revision_link "${mod_rev}" "${repo}" "${pkg}" +# print the given modification revision possibly with a html link to github + +modification_revision_link() { + local mod_rev="$1" + local repo="$2" + local pkg="$3" + + if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then + printf '%s\n' \ + "${mod_rev}" \ + "${repo}" \ + "${pkg}" \ + "${mod_rev}" + else + printf '%s\n' \ + "${mod_rev}" + fi +} + +# trigger_mirror_refreshs +# trigger a refresh of capable tier 1 mirrors (as backup for master mirror) + +trigger_mirror_refreshs() { + local tmp_file + + tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir) + date '+%s' > \ + "${tmp_file}" + ${master_mirror_rsync_command} \ + "${tmp_file}" \ + "${master_mirror_rsync_directory}/lastupdate" + rm "${tmp_file}" + for trigger_url in ${mirror_refresh_trigger_urls}; do + screen -S trigger-mirror-update -d -m curl -L "${trigger_url}" + done +} + +# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name +extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { + pkgname="$1" + pkgname="${pkgname%.pkg.tar.xz}" + arch="${pkgname##*-}" + pkgname="${pkgname%-*}" + sub_pkgrel="${pkgname##*-}" + pkgname="${pkgname%-*}" + pkgrel="${sub_pkgrel%.*}" + if [ "${pkgrel}" = "${sub_pkgrel}" ]; then + sub_pkgrel='0' + else + sub_pkgrel="${sub_pkgrel##*.}" + fi + epoch="${pkgname##*-}" + pkgname="${pkgname%-*}" + pkgver="${epoch#*:}" + if [ "${pkgver}" = "${epoch}" ]; then + epoch='0' + else + epoch="${epoch%%:*}" + fi +} -- cgit v1.2.3-54-g00ecf From 45e7f08ab436087414ba10c862d5f8a59b516d03 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 26 Feb 2018 09:00:53 +0100 Subject: lib/common-functions: save log of executed scripts - for debugging mysterious errors :-) --- conf/default.conf | 4 ++-- lib/common-functions | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'lib/common-functions') diff --git a/conf/default.conf b/conf/default.conf index 18f5d73..7e7ed3d 100755 --- a/conf/default.conf +++ b/conf/default.conf @@ -14,13 +14,13 @@ else base_dir=$(printf '%s/..' "$(dirname "$(readlink -f "$0")")") fi +work_dir="${base_dir}/work" + # shellcheck source=lib/common-functions . "${base_dir}/lib/common-functions" # shellcheck source=lib/mysql-functions . "${base_dir}/lib/mysql-functions" -work_dir="${base_dir}/work" - repo_names='packages community archlinux32' repo_paths__packages="${work_dir}/repos/packages" repo_paths__community="${work_dir}/repos/community" diff --git a/lib/common-functions b/lib/common-functions index a84271b..80f252c 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -15,6 +15,15 @@ if [ -z "${base_dir}" ]; then . 'conf/default.conf' fi +{ + printf 'sourcing common-functions for %s\n' "$0" + printf '%s parameters:' "$#" + printf ' "%s"' "$@" + printf '\n' +} >> \ + "${work_dir}/command-log" + + # find_pkgbuilds package repository git_repository git_revision mod_git_revision # find the PKGBUILD and modification of $package from $repository # sets $PKGBUILD and $PKGBUILD_mod -- cgit v1.2.3-54-g00ecf From 3c876e32b322c15326b5835bb4fef1a02adbcbb6 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 26 Feb 2018 19:38:03 +0100 Subject: lib/common-functions: save date to command-log; skip logging if buildmaster is insane --- lib/common-functions | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions index 80f252c..26151d1 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -15,14 +15,16 @@ if [ -z "${base_dir}" ]; then . 'conf/default.conf' fi -{ - printf 'sourcing common-functions for %s\n' "$0" - printf '%s parameters:' "$#" - printf ' "%s"' "$@" - printf '\n' -} >> \ - "${work_dir}/command-log" - +if [ ! -s "${work_dir}/build-master-sanity" ]; then + { + date + printf 'sourcing common-functions for %s\n' "$0" + printf '%s parameters:' "$#" + printf ' "%s"' "$@" + printf '\n' + } >> \ + "${work_dir}/command-log" +fi # find_pkgbuilds package repository git_repository git_revision mod_git_revision # find the PKGBUILD and modification of $package from $repository -- cgit v1.2.3-54-g00ecf From 6047f2686cfe623e08fbaf4eae0a552ded731eb1 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 08:37:27 +0100 Subject: switch from calling ${mysql_command} to calling the wrapper function mysql_run_query() --- bin/bootstrap-mysql | 4 ++-- bin/build-master-status | 6 +++--- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 4 ++-- bin/copy-to-build-support | 4 ++-- bin/db-update | 6 +++--- bin/get-assignment | 10 +++++----- bin/get-package-updates | 12 ++++++------ bin/modify-package-state | 4 ++-- bin/prioritize-build-list | 2 +- bin/return-assignment | 14 +++++++------- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/slave-build-connect | 2 +- bin/why-dont-you | 2 +- lib/common-functions | 4 ++-- lib/mysql-functions | 34 ++++++++++++++++++++-------------- 17 files changed, 70 insertions(+), 64 deletions(-) (limited to 'lib/common-functions') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index b23bba4..236b0b7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -590,7 +590,7 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repository}" | base64 -w0)" done | \ - ${mysql_command} + mysql_run_query grep '^\('"$( # shellcheck disable=SC2086 @@ -665,7 +665,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then printf ';\n' printf 'DROP TABLE `stable_packages`;\n' } | \ - ${mysql_command} + mysql_run_query fi mysql_repair_binary_packages_without_build_assignment diff --git a/bin/build-master-status b/bin/build-master-status index d74ff8a..9be7823 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -557,12 +557,12 @@ if ${web}; then "${line}" done < \ "${tmp_dir}/todos" | \ - ${mysql_command} + mysql_run_query # insert unfound todos # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / @@ -611,7 +611,7 @@ if ${web}; then printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' printf ');\n' } | \ - ${mysql_command} + mysql_run_query rm -f "${tmp_dir}/todos" fi diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index b55ca65..fa54177 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -48,7 +48,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -72,7 +72,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -90,7 +90,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -121,7 +121,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -156,7 +156,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -190,7 +190,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -210,7 +210,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/bin/check-bugtracker b/bin/check-bugtracker index d6e92e5..f15757b 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do @@ -62,4 +62,4 @@ bug_list=$( printf '"");\n' done done | tee /dev/stderr | \ - ${mysql_command} + mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 808d041..0b87107 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -123,7 +123,7 @@ ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/build-support/" if [ -s "${tmp_dir}/mysql-command" ]; then - ${mysql_command} < \ + mysql_run_query < \ "${tmp_dir}/mysql-command" fi diff --git a/bin/db-update b/bin/db-update index de263be..534c245 100755 --- a/bin/db-update +++ b/bin/db-update @@ -370,7 +370,7 @@ move_packages() { if ${no_action}; then sed 's|^|mysql: |' else - ${mysql_command} + mysql_run_query fi ${dummynator} trigger_mirror_refreshs @@ -457,7 +457,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p @@ -496,7 +496,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p diff --git a/bin/get-assignment b/bin/get-assignment index e3299f6..bc67fab 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -104,7 +104,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query # lock every loop this package breaks find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ @@ -157,7 +157,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) @@ -214,7 +214,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed ' 1d y/\t/ / @@ -238,7 +238,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) diff --git a/bin/get-package-updates b/bin/get-package-updates index 2fd0993..11f384c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -129,7 +129,7 @@ delete_package() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query fi } @@ -144,7 +144,7 @@ clean_up() { printf ' WHERE `repositories`.`name`="to-be-decided"' printf ');\n' } | \ - ${mysql_command} + mysql_run_query } tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) trap 'clean_up' EXIT @@ -353,7 +353,7 @@ done | \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' } | \ sort -u @@ -443,7 +443,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ sort -u } | \ @@ -604,7 +604,7 @@ fi printf ')' printf ' WHERE `from_repo`.`name`="to-be-decided";\n' } | \ - ${mysql_command} + mysql_run_query # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". diff --git a/bin/modify-package-state b/bin/modify-package-state index b0ba53e..8c5fac7 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qxF '1'; then # shellcheck disable=SC2016 @@ -242,7 +242,7 @@ while read -r package reason; do printf ' AND NOT `b`.`is_tested`' printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ - ${mysql_command} + mysql_run_query case "${action}" in 'block') printf '%s' "${reason}" > \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index e8dcf0a..1ff3d5a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -46,7 +46,7 @@ fi "$(printf '%s' "$1" | base64 -w0)" printf ' AND `repositories`.`name`="build-list";\n' } | \ - ${mysql_command} + mysql_run_query # Remove the lock file diff --git a/bin/return-assignment b/bin/return-assignment index db433ee..ed29f34 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -111,7 +111,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' ) for saved_build_log in ${saved_build_logs}; do @@ -164,7 +164,7 @@ if [ "$5" = 'ERROR' ]; then base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query # shellcheck disable=SC2154 sed -i ' @@ -283,7 +283,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -vxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' @@ -441,7 +441,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d ' @@ -513,7 +513,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_ printf ';\n' done } | \ - ${mysql_command} + mysql_run_query # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') @@ -605,7 +605,7 @@ fi printf ');\n' printf 'DROP TABLE `loops_to_delete`;\n' } | \ - ${mysql_command} + mysql_run_query rm -f \ "${work_dir}/package-states/$1.$2.$3.$4.locked" \ "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/bin/seed-build-list b/bin/seed-build-list index 465ef5d..b9ae193 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ @@ -206,7 +206,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r pkgname; do content=$( @@ -294,7 +294,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sort -u | \ sed ' y/\t/ / diff --git a/bin/show-dependencies b/bin/show-dependencies index 4e90258..9e94fe3 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/\s.*$// @@ -178,7 +178,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - ${mysql_command} -N --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 119592e..f8ee9ad 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -29,7 +29,7 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" diff --git a/bin/why-dont-you b/bin/why-dont-you index d95f2a7..b5d73c5 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/lib/common-functions b/lib/common-functions index 26151d1..740c266 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^\S\+\sCONCAT(/d y/\t/ / @@ -680,7 +680,7 @@ remove_old_package_versions() { sed 's/ AND$//' printf ';\n' done | \ - ${mysql_command} + mysql_run_query sed ' s| \(\S\+\)$|-\1| diff --git a/lib/mysql-functions b/lib/mysql-functions index 185d9a9..6cbc502 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -22,6 +22,12 @@ base64_encode_each() { done } +# mysql_run_query +# wrapper function to query mysql +mysql_run_query() { + ${mysql_command} "$@" +} + # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository # shellcheck disable=SC2086 @@ -69,7 +75,7 @@ mysql_add_package_source() { printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ "${values##* }" } | \ - ${mysql_command} + mysql_run_query } # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository @@ -113,7 +119,7 @@ mysql_add_binary_package() { s|, )|)|g s|, FROM| FROM|g ' | \ - ${mysql_command} + mysql_run_query } # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel @@ -162,7 +168,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - ${mysql_command} --html + mysql_run_query --html } # mysql_add_install_target $install_target @@ -180,7 +186,7 @@ mysql_add_install_target() { printf ' VALUES (from_base64("%s"))' \ "${install_target}" } | \ - ${mysql_command} + mysql_run_query } # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository @@ -626,7 +632,7 @@ mysql_generate_package_metadata() { cat "${temp_dir}/add-install-targets-command" fi } | \ - ${mysql_command} + mysql_run_query printf '.' >&2 ) @@ -662,7 +668,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^CONCAT("/d s,^,, @@ -693,7 +699,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s,\t,/, @@ -724,7 +730,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -754,7 +760,7 @@ mysql_find_build_assignment_loops() { "${new_loops}" fi } | \ - ${mysql_command} + mysql_run_query } # mysql_cleanup [dry] @@ -844,7 +850,7 @@ mysql_cleanup() { printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_query_has_pending_dependencies `build_assignment`.`id` @@ -904,7 +910,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r id pkgname arch; do pkgname=$( @@ -933,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qvxF '0'; then pkgbase="${pkgname}" @@ -960,7 +966,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" done | \ - ${mysql_command} + mysql_run_query } # mysql_remove_duplicate_binary_packages @@ -1001,7 +1007,7 @@ mysql_remove_duplicate_build_order() { printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_package_name_query -- cgit v1.2.3-54-g00ecf From 496c571cd233a8646351ba241c3119ceaf2b8063 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 09:05:51 +0100 Subject: use "mysql -N" instead of cumbersome sedding --- bin/build-master-status | 5 ++--- bin/build-master-status-from-mysql | 46 ++++++++++---------------------------- bin/check-bugtracker | 5 ++--- bin/copy-to-build-support | 7 ++---- bin/db-update | 14 ++++-------- bin/get-assignment | 16 +++++-------- bin/get-package-updates | 6 ++--- bin/modify-package-state | 3 +-- bin/return-assignment | 20 +++++------------ bin/seed-build-list | 13 ++++------- bin/show-dependencies | 3 +-- bin/why-dont-you | 7 ++---- lib/common-functions | 7 ++---- lib/mysql-functions | 19 +++++----------- 14 files changed, 51 insertions(+), 120 deletions(-) (limited to 'lib/common-functions') diff --git a/bin/build-master-status b/bin/build-master-status index 9be7823..deb58da 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,10 +562,9 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d - 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' printf ';\n' printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index fa54177..d1985e7 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,11 +27,8 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" sort "${work_dir}/build-list" > \ @@ -48,11 +45,8 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \ @@ -72,11 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -90,11 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -121,9 +107,8 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -156,9 +141,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -190,11 +174,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" { @@ -210,11 +191,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then # TODO: consider versions dependencies, too diff --git a/bin/check-bugtracker b/bin/check-bugtracker index f15757b..156f438 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,8 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' @@ -61,5 +60,5 @@ bug_list=$( tr '\n' ',' printf '"");\n' done - done | tee /dev/stderr | \ + done | \ mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 0b87107..e447596 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,11 +44,8 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${id}" ]; then continue diff --git a/bin/db-update b/bin/db-update index 534c245..78ff622 100755 --- a/bin/db-update +++ b/bin/db-update @@ -457,11 +457,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ sort -u @@ -496,11 +493,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ sort -u diff --git a/bin/get-assignment b/bin/get-assignment index bc67fab..964c6b7 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,11 +58,8 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' { # shellcheck disable=SC2154 @@ -157,8 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ -n "${currently_building}" ]; then @@ -214,9 +210,8 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ + mysql_run_query -N --batch --raw | \ sed ' - 1d y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ ' @@ -238,8 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index 11f384c..e855b19 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,8 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch } | \ sort -u ) @@ -443,8 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 8c5fac7..68b2384 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,8 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index ed29f34..a7e6623 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,11 +70,8 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${infos}" ]; then >&2 echo 'You do not build this package (anymore) - move on.' @@ -111,8 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -283,9 +279,8 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ - grep -vxF '0'; then + mysql_run_query -N --raw --batch | \ + grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi @@ -441,10 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - ' + mysql_run_query -N --raw --batch } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index b9ae193..def9d56 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,9 +125,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' } | \ @@ -206,8 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -294,12 +292,9 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sort -u | \ - sed ' - y/\t/ / - / [0-9a-f]\{40\} [0-9a-f]\{40\} /!d - ' | \ + tr '\t' ' ' | \ if ${update}; then # always block if locked exec 9> "${build_list_lock_file}" diff --git a/bin/show-dependencies b/bin/show-dependencies index 9e94fe3..ed7ca87 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,9 +10,8 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/\s.*$// ' | \ sort -u diff --git a/bin/why-dont-you b/bin/why-dont-you index b5d73c5..35b9419 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,11 +85,8 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' / NULL \S\+$/ b multi-dep diff --git a/lib/common-functions b/lib/common-functions index 740c266..ba434ad 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,11 +501,8 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - /^\S\+\sCONCAT(/d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ shrink_version 4 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 6cbc502..3e7e19f 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -668,9 +668,8 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - /^CONCAT("/d s,^,, s,$,, ' @@ -699,9 +698,8 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s,\t,/, s,\t,-, s,\t,:, @@ -730,11 +728,8 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ { @@ -910,8 +905,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -939,8 +933,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-54-g00ecf From 148d6bbbdee8316cf1ea642ce8615e7c534f6019 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Mar 2018 11:49:50 +0100 Subject: lib/common-functions: PKGBUILD is r/o, so pkgver() won't change the version :-/ --- bin/build-packages | 4 ++-- lib/common-functions | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'lib/common-functions') diff --git a/bin/build-packages b/bin/build-packages index 24c5453..0f0c841 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -352,9 +352,9 @@ while [ "${count}" -ne 0 ]; do if echo "${straw}" | \ grep -qF ':without_check:'; then - innerParameters='--nocheck --holdver' + innerParameters='--nocheck' else - innerParameters='--holdver' + innerParameters='' fi if echo "${straw}" | \ diff --git a/lib/common-functions b/lib/common-functions index ba434ad..0779203 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -934,6 +934,10 @@ extract_source_directory() { # shellcheck disable=SC2016 sed -i '/^\$Id\$$/d' "${output}/PKGBUILD" + # we don't want write permissions on the PKGBUILD - otherwise pkgver() + # will change the version! (**HACK**) + chmod -w "${output}/PKGBUILD" + } # find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository -- cgit v1.2.3-54-g00ecf From 509cb31ae54326d3f8e0a5d9c42f5ef71aa7651b Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Mar 2018 19:43:07 +0100 Subject: lib/common-functions: remove pkgver() from PKGBUILD, so we do not accidentally update $pkgver --- lib/common-functions | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions index 0779203..0f75cb8 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -929,6 +929,10 @@ extract_source_directory() { "${output}/PKGBUILD" fi + # we do not want to update pkgver, so we just undefine it + printf 'unset -f pkgver\n' >> \ + "${output}/PKGBUILD" + mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}" # shellcheck disable=SC2016 -- cgit v1.2.3-54-g00ecf From a13bfbacf27d09f7c8ea6b4b559b0980ee8957b9 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 7 Mar 2018 16:30:10 +0100 Subject: make -N --raw --batch the default arguments to mysql, explicitely override them if required --- bin/build-master-status | 2 +- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 2 +- bin/copy-to-build-support | 2 +- bin/db-update | 4 ++-- bin/get-assignment | 8 ++++---- bin/get-package-updates | 4 ++-- bin/modify-package-state | 2 +- bin/return-assignment | 8 ++++---- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/why-dont-you | 2 +- lib/common-functions | 2 +- lib/mysql-functions | 18 +++++++++--------- 14 files changed, 40 insertions(+), 40 deletions(-) (limited to 'lib/common-functions') diff --git a/bin/build-master-status b/bin/build-master-status index deb58da..006750b 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,7 +562,7 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7cf1b95..a72d396 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" @@ -45,7 +45,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" @@ -66,7 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -80,7 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -107,7 +107,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -141,7 +141,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -174,7 +174,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" @@ -191,7 +191,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 156f438..5bd05b4 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index e447596..b3decd4 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${id}" ]; then diff --git a/bin/db-update b/bin/db-update index 5efa220..5756494 100755 --- a/bin/db-update +++ b/bin/db-update @@ -467,7 +467,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ @@ -503,7 +503,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ diff --git a/bin/get-assignment b/bin/get-assignment index 964c6b7..2df3059 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' { @@ -154,7 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ -n "${currently_building}" ]; then @@ -210,7 +210,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw | \ + mysql_run_query | \ sed ' y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ @@ -233,7 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index e855b19..a95bd94 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -u ) @@ -442,7 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 68b2384..9b0571b 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index a7e6623..717a30e 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${infos}" ]; then @@ -108,7 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query -N --raw --batch + mysql_run_query ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -279,7 +279,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 @@ -436,7 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index def9d56..d49a8f3 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' @@ -205,7 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -292,7 +292,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u | \ tr '\t' ' ' | \ if ${update}; then diff --git a/bin/show-dependencies b/bin/show-dependencies index ed7ca87..492f11b 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/\s.*$// ' | \ @@ -177,7 +177,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/why-dont-you b/bin/why-dont-you index 35b9419..64dfa1b 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' diff --git a/lib/common-functions b/lib/common-functions index 0f75cb8..c7a64fe 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 9d705d2..26d3db8 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -26,17 +26,17 @@ base64_encode_each() { # wrapper function to query mysql mysql_run_query() { if [ -s "${work_dir}/build-master-sanity" ]; then - ${mysql_command} "$@" + ${mysql_command} -N --raw --batch "$@" else { - printf '%s %s: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date } \ | tee -a "${work_dir}/mysql.stdin" \ | tee -a "${work_dir}/mysql.stdout" \ >> "${work_dir}/mysql.stderr" tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} "$@" \ + | ${mysql_command} -N --raw --batch "$@" \ 2>> "${work_dir}/mysql.stderr" \ | tee -a "${work_dir}/mysql.stdout" for s in \ @@ -193,7 +193,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - mysql_run_query --html + mysql_run_query --html --column-names } # mysql_add_install_target $install_target @@ -693,7 +693,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,^,, s,$,, @@ -723,7 +723,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,\t,/, s,\t,-, @@ -753,7 +753,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ @@ -930,7 +930,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -958,7 +958,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-54-g00ecf From 8d6abf54b807e9e1132f7b6dddff0fdf0128aa7c Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 18 Mar 2018 13:30:37 +0100 Subject: lib/common-functions: remove done todo --- lib/common-functions | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions index c7a64fe..1abe7bc 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -4,8 +4,6 @@ # shellcheck disable=SC2039 -# TODO: include link depenendencies in run-depends metadata - # TODO: have full information (currently in files) in database # TODO: remove state files / metadata files -- cgit v1.2.3-54-g00ecf From bd5173dc0c845e0e01de8aed01bd85b0e6646344 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 10:33:35 +0100 Subject: lib/common-functions: irc_say new --- bin/build-master-status-from-mysql | 2 +- bin/get-package-updates | 2 +- bin/return-assignment | 28 +++++++++++++--------------- bin/sanity-check | 4 ++-- lib/common-functions | 8 ++++++++ lib/mysql-functions | 2 +- 6 files changed, 26 insertions(+), 20 deletions(-) (limited to 'lib/common-functions') diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 729375f..e4dd313 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -249,7 +249,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ [ ! -s "${work_dir}/build-master-sanity" ]; then printf 'oh girls, my database is dirty again ...\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say echo 'build master is insane' > \ "${work_dir}/build-master-sanity" fi diff --git a/bin/get-package-updates b/bin/get-package-updates index d3cb140..a6c3f32 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -480,7 +480,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then done } | \ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi exit 3 fi diff --git a/bin/return-assignment b/bin/return-assignment index 417654b..c259f1c 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -222,21 +222,19 @@ if [ "$5" = 'ERROR' ]; then fi ) - if [ -p "${irc_dir}/#archlinux32/in" ]; then - { - printf '%s is broken (says %s).' \ - "$1" \ - "${slave}" - if [ -n "${rescheduled_packages}" ]; then - printf -- ' - I rescheduled:' - # shellcheck disable=SC2086 - printf ' %s,' ${rescheduled_packages} | \ - sed 's/,$/./' - fi - printf '\n' - } | \ - sponge "${irc_dir}/#archlinux32/in" - fi + { + printf '%s is broken (says %s).' \ + "$1" \ + "${slave}" + if [ -n "${rescheduled_packages}" ]; then + printf -- ' - I rescheduled:' + # shellcheck disable=SC2086 + printf ' %s,' ${rescheduled_packages} | \ + sed 's/,$/./' + fi + printf '\n' + } | \ + irc_say fi exit 0 diff --git a/bin/sanity-check b/bin/sanity-check index d98332c..9615ac7 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -22,7 +22,7 @@ usage() { i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION goes insane.\001\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi echo 'build master is insane' > \ "${work_dir}/build-master-sanity" @@ -432,5 +432,5 @@ done if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" printf '\001ACTION resumes sanity.\001\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi diff --git a/lib/common-functions b/lib/common-functions index 1abe7bc..aa32409 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -1390,3 +1390,11 @@ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { epoch="${epoch%%:*}" fi } + +# irc_say +# say content of stdin in irc channel +irc_say() { + if [ -p "${irc_dir}/#archlinux32/in" ]; then + sponge "${irc_dir}/#archlinux32/in" + fi +} diff --git a/lib/mysql-functions b/lib/mysql-functions index 0b21bdf..801eca2 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -83,7 +83,7 @@ mysql_run_query() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_file##*/}" \ - | sponge "${irc_dir}/#archlinux32/in" + | irc_say fi echo 'A mysql query failed.' > \ "${work_dir}/build-master-sanity" -- cgit v1.2.3-54-g00ecf From f22677d62a722fe23e1ca5a75e78cb645efc2bdb Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 14:59:16 +0100 Subject: lib/common-functions: remove unused functions --- lib/common-functions | 109 --------------------------------------------------- 1 file changed, 109 deletions(-) (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions index aa32409..9f78550 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -942,35 +942,6 @@ extract_source_directory() { } -# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository -# return a list of packages on the build list which are (run- / build- / check-time) -# dependencies of the given package - -find_dependencies_on_build_list() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - - generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - - { - cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" - awk '{print $1 "." $2 "." $3 "." $4}' < \ - "${work_dir}/build-list" | \ - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|\.builds| - ' | \ - xargs -r cat | \ - sort -u - } | \ - sort | \ - uniq -d - -} - # download_sources_by_hash $package $repository $git_revision $git_mod_revision # try to download all sources by their hash into the current directory # returns 0 if any source was downloaded and 1 otherwise @@ -1269,86 +1240,6 @@ smoothen_namcap_log() { sponge "${file}" } -# print_list_of_archaic_packages $source1 $source2 ... -# print a list of packages which have not been touched for a while, -# but which are still in the pipeline, e.g. in $source1, $source2 or ... - -print_list_of_archaic_packages() { - for source in "$@"; do - case "${source}" in - 'testing') - # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32 - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \ - -exec head -n1 {} \; | \ - "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin - # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated - # and no longer block other packages from being moved - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \ - sed ' - s|\.testing$|| - ' - ;; - 'build-list') - while read -r pkg rev mod_rev repo; do - git_repo=$( - find_repository_with_commit "${rev}" - ) - eval repo_path='"${repo_paths__'"${git_repo}"'}"' - commit_date=$( - git -C "${repo_path}" show -s --format=%ct "${rev}" - ) - mod_commit_date=$( - git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}" - ) - if [ "${mod_commit_date}" -gt "${commit_date}" ]; then - commit_date="${mod_commit_date}" - fi - # packages remaining longer than $max_package_age_build_list days on the build list - if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then - printf '%s %s %s %s\n' \ - "${pkg}" \ - "${rev}" \ - "${mod_rev}" \ - "${repo}" - fi - done < \ - "${work_dir}/build-list" - ;; - 'staging') - # packages remaining longer than $max_package_age_staging days in staging - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \ - sed ' - s|\.done$|| - ' - ;; - *) - >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}" - ;; - esac - done | \ - sort -u -} - -# modification_revision_link "${mod_rev}" "${repo}" "${pkg}" -# print the given modification revision possibly with a html link to github - -modification_revision_link() { - local mod_rev="$1" - local repo="$2" - local pkg="$3" - - if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then - printf '%s\n' \ - "${mod_rev}" \ - "${repo}" \ - "${pkg}" \ - "${mod_rev}" - else - printf '%s\n' \ - "${mod_rev}" - fi -} - # trigger_mirror_refreshs # trigger a refresh of capable tier 1 mirrors (as backup for master mirror) -- cgit v1.2.3-54-g00ecf From 4465e861c9925f9ed45b4b4229dd712de9915402 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 15:38:16 +0100 Subject: lib/common-functions: decomission even more unused functions --- lib/common-functions | 127 --------------------------------------------------- 1 file changed, 127 deletions(-) (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions index 9f78550..8fc300f 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -1041,133 +1041,6 @@ shrink_version() { ' } -# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ] - -# Return (to stdout) the biggest subset A of the packages in $omega whose -# runtime dependencies in $omega \cup $keep are also in A - -# $all_builds and $all_depends either point to an empty file - then they will get -# filled with cached data for subsequent calls - or to the same files of a previous -# call - -# If non-empty, $force contains packages which are assumed to match the above -# condition without checking. - -# The arguments are names of files with one $package.$revision.$mod_revision.$repository -# per line. - -find_biggest_subset_of_packages() { - - ( # the new shell is intentional - omega="$1" - keep="$2" - all_builds="$3" - all_depends="$4" - if [ $# -eq 4 ]; then - force='/dev/null' - elif [ $# -eq 5 ]; then - force="$5" - else - >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#" - return 2 - fi - - if [ ! -s "${all_builds}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.builds | | - ' {} \; | \ - sort -k2,2 > \ - "${all_builds}" - fi - - if [ ! -s "${all_depends}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.run-depends | | - ' {} \; | \ - grep -v ' base$' | \ - sort -k2,2 > \ - "${all_depends}" - fi - - sort -u "${omega}" | \ - sponge "${omega}" - - temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - { - sort -u "${keep}" - cat "${force}" "${force}" - } | \ - sort | \ - uniq -u > \ - "${temp_dir}/keep.new" - touch "${temp_dir}/keep" - - while [ -s "${temp_dir}/keep.new" ]; do - cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \ - sort -u | \ - sponge "${temp_dir}/keep" - - { - # we append all packages which are run-dependencies of keep-packages - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.run-depends| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.run-depends:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_builds}" - - # we append all packages with run-dependencies on the keep-list - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.builds:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_depends}" - } | \ - sort -u | \ - join -1 1 -2 1 -o 2.1 - "${omega}" | \ - sort -u > \ - "${temp_dir}/keep.new" - - # "new" is only what has not been there before and what is not forced - cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \ - sort | \ - uniq -u | \ - sponge "${temp_dir}/keep.new" - done - - cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \ - sort | \ - uniq -u - - ) - -} - # sort_square_bracket_content $file # sort the content of [] in $file, print to stdout -- cgit v1.2.3-54-g00ecf From 7cc6dee8cfd91e44e7d6436e824937a8a0add008 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 14:15:36 +0100 Subject: lib/common-functions: silence shellcheck --- lib/common-functions | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions index 8fc300f..e3aab4c 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -908,6 +908,7 @@ find_package_repository_to_package() { extract_source_directory() { local git_repo="$1" + # shellcheck disable=SC2034 local rev="$2" local mod_rev="$3" local output="$4" -- cgit v1.2.3-54-g00ecf From 4f5f0fafc4be959a9a05adac8bf5162a7c5bab46 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 14:18:41 +0100 Subject: lib/common-functions: remove generate_package_metadata, delete_old_metadata - this information is now in the database --- bin/cleanup | 3 - bin/get-package-updates | 1 - bin/seed-build-list | 1 - lib/common-functions | 228 +----------------------------------------------- 4 files changed, 1 insertion(+), 232 deletions(-) (limited to 'lib/common-functions') diff --git a/bin/cleanup b/bin/cleanup index 15b6fc9..02ac98b 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -99,9 +99,6 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \ -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \ -delete -# remove old package meta data -delete_old_metadata - # remove dependency graphs of packages on the deletion list sed ' s|^|'"${webserver_directory}"'/graphs/| diff --git a/bin/get-package-updates b/bin/get-package-updates index ce9d57a..a36cb8a 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -353,7 +353,6 @@ done | \ } | \ mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" - generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi diff --git a/bin/seed-build-list b/bin/seed-build-list index 0db9cdf..41707dd 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -373,7 +373,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2 printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \ sed 's/ $/\n/' - generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \ "${work_dir}/build-list" diff --git a/lib/common-functions b/lib/common-functions index e3aab4c..0a5ef1c 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -6,7 +6,7 @@ # TODO: have full information (currently in files) in database -# TODO: remove state files / metadata files +# TODO: remove state files if [ -z "${base_dir}" ]; then # just to make shellcheck happy @@ -123,232 +123,6 @@ find_git_repository_to_package_repository() { } -# generate_package_metadata $package $git_revision $mod_git_revision $repository -# or -# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# generate the meta data files of a package (dependencies, built packages, ...) - -generate_package_metadata() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - local file_prefix - local file - local PKGBUILD - - if [ $# -eq 1 ]; then - # second form - repository="${package##*.}" - package="${package%.*}" - mod_git_revision="${package##*.}" - package="${package%.*}" - git_revision="${package##*.}" - package="${package%.*}" - fi - - file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}" - - if [ -e "${file_prefix}.builds" ] && \ - [ -e "${file_prefix}.build-depends" ] && \ - [ -e "${file_prefix}.run-depends" ] && \ - [ -e "${file_prefix}.groups" ] && \ - [ -e "${file_prefix}.packages" ]; then - return 0 - fi - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then - printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO" - exit 1 - fi - if [ ! -s "${file_prefix}.SRCINFO" ]; then - >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO" - exit 1 - fi - - # otherwise this just calls for trouble - sed -i ' - /^[^=]*=\s*$/d - s/_i686\(\s*=\)/\1/ - ' "${file_prefix}.SRCINFO" - - # extract "groups" = groups \cup provides - grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.groups" - - # extract "packages" = pkgname - grep '^pkgname = ' "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.packages" - - # extract "builds" = provides \cup pkgname \cup groups - cat "${file_prefix}.groups" "${file_prefix}.packages" | \ - sort -u > \ - "${file_prefix}.builds" - - # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' 'base-devel' - sed -n "$( - printf '/^\t%s = /p\n' \ - 'depends' \ - 'makedepends' - )" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.build-depends" - - # extract "run-depends" = depends \cup \{ base \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' - sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.run-depends" - - rm "${file_prefix}.SRCINFO" - -} - -# delete_old_metadata -# delete old (=unneeded) meta data of packages - -delete_old_metadata() { - - local current_metadata - - current_metadata=$( - find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.[^.]\+$|| - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3| - ' | \ - sort -u - ) - - ( # the new shell is intentional - # what we have - echo "${current_metadata}" - - # package-states should stay - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - ' | \ - sort -u | \ - sed 'p' - - # build-list items should stay - sed 'p' "${work_dir}/build-list" - - tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - echo "${current_metadata}" | \ - sort -k1,1 > \ - "${tmp_dir}/current-metadata" - - # the newest of the following should stay: - { - # deletion-list items - cat "${work_dir}/deletion-list" - # all packages in the repos - for repo in ${repo_names}; do - eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \ - tar -t | \ - sed ' - s|/$|| - /\//d - ' - done - } | \ - sort -u | \ - join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \ - sort -k4,4 > \ - "${tmp_dir}/find-newest-revisions" - - uniq -uf3 < \ - "${tmp_dir}/find-newest-revisions" | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - - uniq -Df3 < \ - "${tmp_dir}/find-newest-revisions" | \ - uniq --group=append -f3 | \ - { - revs='' - mod_revs='' - opkg='' - orepo='' - while read -r rev mod_rev repo pkg; do - - if [ -z "${rev}" ] && \ - [ -z "${mod_rev}" ] && \ - [ -z "${repo}" ] && \ - [ -z "${pkg}" ]; then - - printf '%s %s %s %s\n' \ - "$( - printf '%s\n' ${revs} | \ - find_newest_of_git_revisions - )" \ - "$( - printf '%s\n' ${mod_revs} | \ - find_newest_of_git_revisions - )" \ - "${orepo}" \ - "${opkg}" - - revs='' - mod_revs='' - orepo='' - opkg='' - continue - fi - revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${revs} ${rev} | \ - sort -u - ) - mod_revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${mod_revs} ${mod_rev} | \ - sort -u - ) - orepo="${repo}" - opkg="${pkg}" - done - } | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - ) | \ - sort | \ - uniq -u | \ - while read -r pkg rev mod_rev repo; do - rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."* - done -} - # repository_of_package $package.$repo_revision.$mod_repo_revision.$repository # print which (stable) repository a package belongs to -- cgit v1.2.3-54-g00ecf From c772be57eca3a2176ac3931c02fb582f85c0d575 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 15:19:23 +0100 Subject: decommission package-states directory - we rely on the database now --- bin/bootstrap-mysql | 60 ++++------------------------------------------ bin/cleanup | 20 ---------------- bin/db-update | 62 ------------------------------------------------ bin/get-assignment | 27 +++------------------ bin/get-package-updates | 6 ----- bin/modify-package-state | 38 ----------------------------- bin/return-assignment | 38 ----------------------------- lib/common-functions | 4 ---- 8 files changed, 7 insertions(+), 248 deletions(-) (limited to 'lib/common-functions') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 4c1938f..fc347d7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -3,6 +3,10 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: this should become some sort of mysql dump to bootstrap the +# tables and stored functions only - when the database is gone, we're +# pretty much screwed anyway :-D + tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT @@ -792,68 +796,12 @@ fi } | \ mysql_run_query -find "${work_dir}/package-states" \ - -maxdepth 1 \ - -mindepth 1 \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed ' - 1!d - s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@ - s@^\S*/@@ - s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 / - s/ \([0-9]\+\) / \1.0 / - s/ [0-9]\+\.\([0-9]\+\) / \1 / - ' {} \; | \ - while read -r state_file sub_pkgrel arch package_file; do - printf '%s ' "${state_file}" - repository=$( - grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \ - cut -d' ' -f1 - ) - if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then - printf 'not found exactly once on the master mirror.\n' - >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}" - continue - fi - mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}" - printf ' ok\n' - done - tr ' ' '.' < \ "${work_dir}/build-list" | \ while read -r state_file; do mysql_generate_package_metadata 'build-list' "${state_file}" done -# shellcheck disable=SC2016 -find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ - sed ' - s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3| - ' | \ - while read -r state_file pkgbase git_revision mod_git_revision repository; do - printf 'UPDATE `build_assignments`' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ - "$( - tr -d '\n' < \ - "${state_file}" | \ - base64 -w0 - )" - printf ' WHERE' - printf ' `package_sources`.`%s`=from_base64("%s") AND' \ - 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)" - printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ - "$(printf '%s' "${repository}" | base64 -w0)" - done | \ - mysql_run_query - grep '^\('"$( # shellcheck disable=SC2086 printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories} diff --git a/bin/cleanup b/bin/cleanup index a0bf790..7aaa39f 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -19,26 +19,6 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -# remove blocked/broken/locked markes of packages not on the buildlist anymore - -{ - find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \ - sed ' - s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5| - ' - awk '{ - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked" - }' "${work_dir}/build-list" -} | \ - sort -k3 | \ - uniq -uf2 | \ - grep '^state ' | \ - awk '{print $2}' | \ - sed "s|^|${work_dir}/package-states/|" | \ - xargs -rn1 rm - # remove logs where package is not broken/locked anymore { diff --git a/bin/db-update b/bin/db-update index 6ecf9c9..6722f24 100755 --- a/bin/db-update +++ b/bin/db-update @@ -207,12 +207,6 @@ for source_stability in 'testing' 'staging'; do mysql_join_repositories_repository_stabilities printf ' AND `repository_stabilities`.`name`="%s"' \ "${source_stability}" - # TODO: we may relax this condition, if we a) got rid of state files - # or b) improved keeping them in sync (*.testing -> /dev/null is not - # implemented, only .tested -> /dev/null) - if [ ! "${source_stability}" = 'staging' ]; then - printf ' AND `binary_packages`.`is_tested`' - fi mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories @@ -401,62 +395,6 @@ for source_stability in 'testing' 'staging'; do printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n' } | \ mysql_run_query - - sed_apply_rm=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/rm" - ) - - sed_apply_remove_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/mv" - ) - - sed_apply_move_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/{\n' "$(str_to_regex "${pkg}")" - printf 'w %%s\n' - printf 'd\n' - printf '}\n' - done < \ - "${tmp_dir}/mv" - ) - - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed -i "${sed_apply_rm}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.tested' \ - -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.done' \ - -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \ - -printf '%p\n' | \ - while read -r sf; do - sed -i "$( - printf '%s\n' "${sed_apply_move_part_of_mv}" | \ - sed ' - s,%s,'"${sf%.done}.testing"',g - ' - )" "${sf}" - done - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -type f \ - -not -exec test -s '{}' \; \ - -delete done trigger_mirror_refreshs diff --git a/bin/get-assignment b/bin/get-assignment index 3357fc6..a82e1f8 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -18,17 +18,8 @@ # TODO: honor manual build order of tool-chain: # toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc -mkdir -p "${work_dir}/package-states" - hand_out_assignment() { - # we don't care anymore if an older version of this package was - # "locked" or "broken" - find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \ - -delete - # move that build order to the end of the build-list sed -i ' /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { @@ -57,21 +48,7 @@ hand_out_assignment() { printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' - } | \ - mysql_run_query | \ - tr '\t' ' ' - { - # shellcheck disable=SC2154 - echo "${slave}" - if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - cat "${work_dir}/package-states/$1.$2.$3.$4.locked" - fi - } | \ - sort -u | \ - sponge "${work_dir}/package-states/$1.$2.$3.$4.locked" - # shellcheck disable=SC2016 - { printf 'UPDATE `build_slaves`' printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' @@ -86,6 +63,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ')' + # shellcheck disable=SC2154 printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${slave}" | base64 -w0)" @@ -101,7 +79,8 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - mysql_run_query + mysql_run_query | \ + tr '\t' ' ' exit 0 diff --git a/bin/get-package-updates b/bin/get-package-updates index a36cb8a..3e9edf9 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -111,9 +111,6 @@ delete_package() { mysql_run_query >> \ "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \ - -delete # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. @@ -317,9 +314,6 @@ done | \ if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then # new or modified PKGBUILD sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -regextype grep \ - -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \ - -delete # shellcheck disable=SC2154 echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ "${work_dir}/build-list.new" diff --git a/bin/modify-package-state b/bin/modify-package-state index 76b8c1e..4a2f5ac 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -243,46 +243,8 @@ while read -r package reason; do { printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" printf ';\n' - # TODO: remove the below part, once the database if fully functional - printf 'UPDATE `binary_packages` AS `a`' - printf ' JOIN `binary_packages` AS `b`' - printf ' ON `a`.`build_assignment`=`b`.`build_assignment`' - mysql_join_binary_packages_repositories 'b' - printf ' SET `b`.`is_tested`=1' - printf ' WHERE `a`.`is_tested`' - printf ' AND NOT `b`.`is_tested`' - printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ mysql_run_query - case "${action}" in - 'block') - printf '%s' "${reason}" > \ - "${work_dir}/package-states/${package}.blocked" - ;; - 'unblock') - rm -f "${work_dir}/package-states/${package}.blocked" - ;; - 'faulty') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.tested' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ed}ing" - done - ;; - 'tested') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.testing' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ing}ed" - done - ;; - *) - >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}" - exit 42 - ;; - esac printf '%s %s\n' "${action}" "${package}" fi done \ diff --git a/bin/return-assignment b/bin/return-assignment index 02bd491..da9e762 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -89,9 +89,6 @@ if [ "$5" = 'ERROR' ]; then '*.build-log.gz' ) - # shellcheck disable=SC2154 - echo "${slave}" >> \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" # shellcheck disable=SC2016 { if [ -n "${saved_build_logs}" ]; then @@ -162,23 +159,6 @@ if [ "$5" = 'ERROR' ]; then } | \ mysql_run_query - # shellcheck disable=SC2154 - sed -i ' - /^'"$(str_to_regex "${slave}")"'$/d - ' "${work_dir}/package-states/$1.$2.$3.$4.locked" - if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - rm "${work_dir}/package-states/$1.$2.$3.$4.locked" - - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - fi - # release lock on build-list - otherwise seed-build-list won't run flock -u 9 @@ -517,24 +497,9 @@ for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done -# remove old state files (these should be only "done" markers, but -# actually we don't care what it is) - as long as it's not "testing" or "tested" -find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -not -name '*.testing' \ - -not -name '*.tested' \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \ - -exec rm '{}' \; - # remove package from build list sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" -# remove package lock file -if ! [ "${destination}" = 'build-support' ]; then - # shellcheck disable=SC2086 - printf '%s\n' ${packages} > \ - "${work_dir}/package-states/$1.$2.$3.$4.done" -fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' @@ -578,6 +543,3 @@ fi printf 'DROP TABLE `loops_to_delete`;\n' } | \ mysql_run_query -rm -f \ - "${work_dir}/package-states/$1.$2.$3.$4.locked" \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/lib/common-functions b/lib/common-functions index 0a5ef1c..f66ccd1 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -4,10 +4,6 @@ # shellcheck disable=SC2039 -# TODO: have full information (currently in files) in database - -# TODO: remove state files - if [ -z "${base_dir}" ]; then # just to make shellcheck happy . 'conf/default.conf' -- cgit v1.2.3-54-g00ecf From aa91222717dabe20642aeb8ddf9b2a127ba19f0f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 23 Mar 2018 09:57:21 +0100 Subject: lib/common-functions: find_git_repository_to_package_repository read info from database instead of git --- lib/common-functions | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) (limited to 'lib/common-functions') diff --git a/lib/common-functions b/lib/common-functions index f66ccd1..8425e9a 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -88,35 +88,25 @@ find_repository_with_commit() { find_git_repository_to_package_repository() { local repository - local package_repository - local repo_path - - package_repository="$1" - if [ "$1" = 'build-support' ]; then - echo 'packages' + repository=$( + # shellcheck disable=SC2016 + { + printf 'SELECT `git_repositories`.`name` FROM `git_repositories`' + mysql_join_git_repositories_upstream_repositories + printf ' WHERE `upstream_repositories`.`name`=from_base64("%s");\n' \ + "$(printf '%s' "$1" | base64 -w0)" + } | \ + mysql_run_query + ) + if [ -z "${repository}" ]; then + >&2 echo "can't find git repository with package repository '$1'" + exit 1 + else + echo "${repository}" return 0 fi - for repository in ${repo_names}; do - if [ "${repository}" = "archlinux32" ]; then - continue - fi - eval 'repo_path="${repo_paths__'"${repository}"'}"' - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \ - tar -t --wildcards '*/repos' | \ - grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \ - cut -d/ -f3 | \ - sed 's|-[^-]\+$||' | \ - sort -u | \ - grep -qxF "${package_repository}"; then - echo "${repository}" - return 0 - fi - done - >&2 echo "can't find git repository with package repository '$1'" - exit 1 - } # repository_of_package $package.$repo_revision.$mod_repo_revision.$repository -- cgit v1.2.3-54-g00ecf