summaryrefslogtreecommitdiff
path: root/bin
diff options
context:
space:
mode:
Diffstat (limited to 'bin')
-rwxr-xr-xbin/common-functions1378
-rwxr-xr-xbin/mysql-functions1098
2 files changed, 0 insertions, 2476 deletions
diff --git a/bin/common-functions b/bin/common-functions
deleted file mode 100755
index a84271b..0000000
--- a/bin/common-functions
+++ /dev/null
@@ -1,1378 +0,0 @@
-#!/bin/sh
-
-# contains functions used by more than one script
-
-# shellcheck disable=SC2039
-
-# TODO: include link depenendencies in run-depends metadata
-
-# TODO: have full information (currently in files) in database
-
-# TODO: remove state files / metadata files
-
-if [ -z "${base_dir}" ]; then
- # just to make shellcheck happy
- . 'conf/default.conf'
-fi
-
-# find_pkgbuilds package repository git_repository git_revision mod_git_revision
-# find the PKGBUILD and modification of $package from $repository
-# sets $PKGBUILD and $PKGBUILD_mod
-
-find_pkgbuilds() {
-
- local package="$1"
- local repository="$2"
- local git_repository="$3"
- local git_revision="$4"
- local mod_git_revision="$5"
-
- local repo_path
- eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
- PKGBUILD=$(
- git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \
- tar -t 2> /dev/null | \
- grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \
- grep -v -- '-i686/PKGBUILD$' | \
- grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
- sort | \
- tail -n1
- )
-
- PKGBUILD_mod=$(
- git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \
- tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null
- ) || true
-
- if [ -z "${PKGBUILD}" ] && \
- [ -z "${PKGBUILD_mod}" ]; then
- >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \
- "${package}" \
- "${repository}" \
- "${git_repository}" \
- "${git_revision}" \
- "${mod_git_revision}"
- return 1
- fi
-
-}
-
-# find_repository_with_commit commit
-# find the repository which has $commit
-
-find_repository_with_commit() {
-
- local repository
-
- for repository in ${repo_names}; do
- # shellcheck disable=SC2016
- if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then
- echo "${repository}"
- return 0
- fi
- done
- >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1"
- exit 1
-
-}
-
-# find_git_repository_to_package_repository repository
-# find the git repository which tracks the package repository $repository
-
-find_git_repository_to_package_repository() {
-
- local repository
- local package_repository
- local repo_path
-
- package_repository="$1"
-
- if [ "$1" = 'build-support' ]; then
- echo 'packages'
- return 0
- fi
-
- for repository in ${repo_names}; do
- if [ "${repository}" = "archlinux32" ]; then
- continue
- fi
- eval 'repo_path="${repo_paths__'"${repository}"'}"'
- if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \
- tar -t --wildcards '*/repos' | \
- grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \
- cut -d/ -f3 | \
- sed 's|-[^-]\+$||' | \
- sort -u | \
- grep -qxF "${package_repository}"; then
- echo "${repository}"
- return 0
- fi
- done
- >&2 echo "can't find git repository with package repository '$1'"
- exit 1
-
-}
-
-# generate_package_metadata $package $git_revision $mod_git_revision $repository
-# or
-# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# generate the meta data files of a package (dependencies, built packages, ...)
-
-generate_package_metadata() {
-
- local package="$1"
- local git_revision="$2"
- local mod_git_revision="$3"
- local repository="$4"
- local file_prefix
- local file
- local PKGBUILD
-
- if [ $# -eq 1 ]; then
- # second form
- repository="${package##*.}"
- package="${package%.*}"
- mod_git_revision="${package##*.}"
- package="${package%.*}"
- git_revision="${package##*.}"
- package="${package%.*}"
- fi
-
- file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
-
- if [ -e "${file_prefix}.builds" ] && \
- [ -e "${file_prefix}.build-depends" ] && \
- [ -e "${file_prefix}.run-depends" ] && \
- [ -e "${file_prefix}.groups" ] && \
- [ -e "${file_prefix}.packages" ]; then
- return 0
- fi
-
- if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then
- printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"
- exit 1
- fi
- if [ ! -s "${file_prefix}.SRCINFO" ]; then
- >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO"
- exit 1
- fi
-
- # otherwise this just calls for trouble
- sed -i '
- /^[^=]*=\s*$/d
- s/_i686\(\s*=\)/\1/
- ' "${file_prefix}.SRCINFO"
-
- # extract "groups" = groups \cup provides
- grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u > \
- "${file_prefix}.groups"
-
- # extract "packages" = pkgname
- grep '^pkgname = ' "${file_prefix}.SRCINFO" | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u > \
- "${file_prefix}.packages"
-
- # extract "builds" = provides \cup pkgname \cup groups
- cat "${file_prefix}.groups" "${file_prefix}.packages" | \
- sort -u > \
- "${file_prefix}.builds"
-
- # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
- {
- {
- printf 'all_depend = %s\n' 'base' 'base-devel'
- sed -n "$(
- printf '/^\t%s = /p\n' \
- 'depends' \
- 'makedepends'
- )" "${file_prefix}.SRCINFO"
- } | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u
- sed 'p' "${file_prefix}.builds"
- } | \
- sort | \
- uniq -u > \
- "${file_prefix}.build-depends"
-
- # extract "run-depends" = depends \cup \{ base \} \setminus "builds"
- {
- {
- printf 'all_depend = %s\n' 'base'
- sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO"
- } | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u
- sed 'p' "${file_prefix}.builds"
- } | \
- sort | \
- uniq -u > \
- "${file_prefix}.run-depends"
-
- rm "${file_prefix}.SRCINFO"
-
-}
-
-# delete_old_metadata
-# delete old (=unneeded) meta data of packages
-
-delete_old_metadata() {
-
- local current_metadata
-
- current_metadata=$(
- find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
- sed '
- s|\.[^.]\+$||
- s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
- ' | \
- sort -u
- )
-
- ( # the new shell is intentional
- # what we have
- echo "${current_metadata}"
-
- # package-states should stay
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
- sed '
- s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
- ' | \
- sort -u | \
- sed 'p'
-
- # build-list items should stay
- sed 'p' "${work_dir}/build-list"
-
- tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
- echo "${current_metadata}" | \
- sort -k1,1 > \
- "${tmp_dir}/current-metadata"
-
- # the newest of the following should stay:
- {
- # deletion-list items
- cat "${work_dir}/deletion-list"
- # all packages in the repos
- for repo in ${repo_names}; do
- eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \
- tar -t | \
- sed '
- s|/$||
- /\//d
- '
- done
- } | \
- sort -u | \
- join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
- sort -k4,4 > \
- "${tmp_dir}/find-newest-revisions"
-
- uniq -uf3 < \
- "${tmp_dir}/find-newest-revisions" | \
- awk '{print $4 " " $1 " " $2 " " $3}' | \
- sed 'p'
-
- uniq -Df3 < \
- "${tmp_dir}/find-newest-revisions" | \
- uniq --group=append -f3 | \
- {
- revs=''
- mod_revs=''
- opkg=''
- orepo=''
- while read -r rev mod_rev repo pkg; do
-
- if [ -z "${rev}" ] && \
- [ -z "${mod_rev}" ] && \
- [ -z "${repo}" ] && \
- [ -z "${pkg}" ]; then
-
- printf '%s %s %s %s\n' \
- "$(
- printf '%s\n' ${revs} | \
- find_newest_of_git_revisions
- )" \
- "$(
- printf '%s\n' ${mod_revs} | \
- find_newest_of_git_revisions
- )" \
- "${orepo}" \
- "${opkg}"
-
- revs=''
- mod_revs=''
- orepo=''
- opkg=''
- continue
- fi
- revs=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${revs} ${rev} | \
- sort -u
- )
- mod_revs=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${mod_revs} ${mod_rev} | \
- sort -u
- )
- orepo="${repo}"
- opkg="${pkg}"
- done
- } | \
- awk '{print $4 " " $1 " " $2 " " $3}' | \
- sed 'p'
- ) | \
- sort | \
- uniq -u | \
- while read -r pkg rev mod_rev repo; do
- rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
- done
-}
-
-# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
-# print which (stable) repository a package belongs to
-
-repository_of_package() {
- local package="$1"
- local repository="${package##*.}"
- package="${package%.*}"
- local a32_rev="${package##*.}"
- package="${package%.*.*}"
-
- case "${repository}" in
- 'multilib')
- if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \
- tar -Ox | \
- grep -qFx "${package%.*.*.*}"; then
- echo 'extra'
- else
- echo 'community'
- fi
- ;;
- *)
- echo "${repository}"
- esac
-}
-
-# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending
-# print wether the specified package is an official package (print
-# $ending) or a community package (print 'community-$ending') or a
-# build-support package (print 'build-support')
-
-official_or_community() {
- local prepo
- prepo=$(repository_of_package "$1")
-
- if [ "${prepo}" = 'community' ]; then
- echo 'community-'"$2"
- elif [ "${prepo}" = 'build-support' ]; then
- echo 'build-support'
- else
- echo "$2"
- fi
-}
-
-# ls_master_mirror $path
-# list content of $path on the master mirror (via rsync)
-
-ls_master_mirror() {
-
- local path="$1"
-
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/${path}/" | \
- grep -v '\s\.$' | \
- awk '{print $5}'
-
-}
-
-# TODO: the actions of remove_old_package_versions should be done
-# on basis of the information in the database
-
-# remove_old_package_versions $arch $repository $package_file
-
-# removes all older (not-newer) versions of $package_file
-# in all repositories not-older (newer) than $repository
-
-# TODO: should remove all other version (also newer) from
-# some repositories :-/
-
-# A package is considered not newer if
-# a) its version is not newer
-# A package is considered older if
-# b) its version is older or
-# c) if it's "not newer" and its architecture is 'any' and different or
-# d) if it's "not newer" and the other architecture is 'any' and different
-
-# this ensures an any package may replace arch-specific packages of the same version and vice versa
-
-remove_old_package_versions() {
-
- local arch="$1"
- local repository="$2"
- local package="$3"
-
- local pkgname
- local epoch
- local pkgver
- local pkgrel
- local sub_pkgrel
- pkgname="${package%-*}"
- pkgrel="${pkgname##*-}"
- sub_pkgrel="${pkgrel##*.}"
- if [ "${sub_pkgrel}" = "${pkgrel}" ]; then
- sub_pkgrel='0'
- else
- pkgrel="${pkgrel%.*}"
- fi
- pkgname="${pkgname%-*}"
- pkgver="${pkgname##*-}"
- epoch="${pkgver%%:*}"
- if [ "${epoch}" = "${pkgver}" ]; then
- epoch='0'
- else
- pkgver="${pkgver#*:}"
- fi
- pkgname="${pkgname%-*}"
-
- # shellcheck disable=SC2016
- {
- printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \
- "$(
- printf '%s' "${repository}" | \
- base64 -w0
- )" \
- "$(
- printf '%s' "${package}" | \
- base64 -w0
- )" \
- "$(
- printf '%s' "${package}" | \
- sed '
- s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/
- ' | \
- base64 -w0
- )"
- printf 'SELECT '
- printf '`binary_packages`.`id`,'
- printf 'CONCAT(`repositories`.`name`,"/",'
- mysql_package_name_query
- printf '),'
- # should we delete packages of identical version?
- printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),'
- printf 'CONCAT('
- printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
- printf '`binary_packages`.`pkgver`,"-",'
- printf '`binary_packages`.`pkgrel`,".",'
- printf '`binary_packages`.`sub_pkgrel`'
- printf ')'
- printf ' FROM `binary_packages`'
- mysql_join_binary_packages_repositories
- mysql_join_binary_packages_architectures
- printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
- printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`'
- # name must match
- printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \
- "$(printf '%s' "${package%-*-*-*}" | base64 -w0)"
- # repository, where package should be deleted, should be less stable
- printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \
- "$(printf '%s' "${repository}" | base64 -w0)"
- printf ';\n'
- } | \
- ${mysql_command} --raw --batch | \
- sed '
- /^\S\+\sCONCAT(/d
- y/\t/ /
- ' | \
- expand_version 4 | \
- sort -k4V,4 -k3r,3 | \
- shrink_version 4 | \
- sed -n '
- /^bogus /q
- p
- ' | \
- cut -d' ' -f1,2 >&2
-
- # repositories in which older packages should be deleted
- local delete_older_repositories
- # repositories in which not-newer packages should be deleted
- local delete_not_newer_repositories
-
- if echo "${standalone_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories="${repository}"
- delete_not_newer_repositories=''
-
- elif echo "${staging_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories="${repository}"
- delete_not_newer_repositories=$(
- echo "${staging_package_repositories}" | \
- grep -vxF "${repository}"
- ) || true
-
- elif echo "${testing_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories=$(
- printf '%s\n' "${staging_package_repositories}" "${repository}"
- )
- delete_not_newer_repositories=$(
- echo "${testing_package_repositories}" | \
- grep -vxF "${repository}"
- ) || true
-
- elif echo "${stable_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories=$(
- printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}"
- )
- delete_not_newer_repositories=$(
- echo "${stable_package_repositories}" | \
- grep -vxF "${repository}"
- ) || true
-
- else
-
- >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}"
- return 1
-
- fi
-
- ( # the new shell is intentional
- tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
- {
- # the architecture of the package (any vs. i686)
- package_arch="${package##*-}"
- package_arch="${package_arch%%.*}"
- if [ "${package_arch}" = 'any' ]; then
- package_arch_regex_inverter='!'
- else
- unset package_arch_regex_inverter
- fi
-
- for repo in ${delete_older_repositories}; do
- ls_master_mirror "${arch}/${repo}" | \
- sed -n '
- /\.pkg\.tar\.xz$/!d
- s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
- /^'"$(str_to_regex "${package%-*-*-*}")"' / {
- s|^|2 '"${arch} ${repo}"' |
- / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{
- s|^2|0|
- }
- p
- }
- '
- done
- for repo in ${delete_not_newer_repositories}; do
- ls_master_mirror "${arch}/${repo}" | \
- sed -n '
- /\.pkg\.tar\.xz$/!d
- s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
- /^'"$(str_to_regex "${package%-*-*-*}")"' / {
- s|^|0 '"${arch} ${repo}"' |
- p
- }
- '
- done
- echo "${package%-*}" | \
- sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|'
-
- # the generated list contains the following columns:
- # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz
- } | \
- expand_version 5 | \
- sort -k5V,5 -k1n,1 | \
- shrink_version 5 | \
- sed -n '
- /^1 %cut% %it% %here% /q
- s/^[02] //
- s/ \(\S\+\)$/-\1/
- p
- ' | \
- sort -u > \
- "${tmp_dir}/packages-to-delete"
- # this file contains a list of packages to be deleted, one on each line:
- # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz
-
- cut -d' ' -f1,2 < \
- "${tmp_dir}/packages-to-delete" | \
- grep -vxF "${arch} ${repository}" | \
- sort -u > \
- "${tmp_dir}/repositories-to-modify"
-
- # fetch all databases being modified
- while read -r del_arch del_repo; do
- mkdir -p "${tmp_dir}/${del_arch}/${del_repo}"
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \
- "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \
- "${tmp_dir}/${del_arch}/${del_repo}/"
- done < \
- "${tmp_dir}/repositories-to-modify"
-
- while read -r del_arch del_repo del_package _; do
- if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then
- # we do not repo-remove the package in the target repository
- continue
- fi
- repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \
- "${del_package}"
- done < \
- "${tmp_dir}/packages-to-delete"
-
- # upload modified databases
- while read -r del_arch del_repo; do
- ${master_mirror_rsync_command} \
- "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \
- "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \
- "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/"
- done < \
- "${tmp_dir}/repositories-to-modify"
-
- # shellcheck disable=SC2016
- sed '
- s/\.pkg\.tar\.xz$//
- s/^\S\+ //
- s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/
- s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/
- s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/
- ' "${tmp_dir}/packages-to-delete" | \
- while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do
- printf 'DELETE FROM `binary_packages` WHERE'
- printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \
- 'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \
- 'repository' 'repositories' 'repositories' 'repositories' "$(printf '%s' "${repo}" | base64 -w0)"
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \
- 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
- 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
- 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
- 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \
- sed 's/ AND$//'
- printf ';\n'
- done | \
- ${mysql_command}
-
- sed '
- s| \(\S\+\)$|-\1|
- y| |/|
- s|^|rm "|
- s|$|"|
- p
- s|"$|.sig"|
- ' "${tmp_dir}/packages-to-delete" | \
- ${master_mirror_sftp_command}
- )
-
-}
-
-# wait_some_time $minimum $diff
-# wait between minimum and minimum+diff seconds (diff defaults to 30)
-
-wait_some_time() {
- local minimum=$1
- local diff=$2
- local random
-
- if [ -z "${diff}" ]; then
- diff=30
- fi
-
- random=$(
- dd if='/dev/urandom' count=1 2> /dev/null | \
- cksum | \
- cut -d' ' -f1
- )
-
- sleep $((minimum + random % diff))
-}
-
-# str_to_regex $string
-# escape dots for use in regex
-
-str_to_regex() {
- echo "$1" | \
- sed '
- s|[.[]|\\\0|g
- '
-}
-
-# make_source_info $package $repository $git_revision $mod_git_revision $output
-# create .SRCINFO from PKGBUILD within git repositories, output to $output
-
-make_source_info() {
-
- local package="$1"
- local repository="$2"
- local git_revision="$3"
- local mod_git_revision="$4"
- local output="$5"
-
- local git_repo
- local PKGBUILD
- local PKGBUILD_mod
-
- git_repo=$(find_repository_with_commit "${git_revision}")
-
- if [ -z "${git_repo}" ]; then
- return 1
- fi
-
- find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
-
- ( # the new shell is intentional
-
- tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX")
- trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
- extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0'
-
- {
- cd "${tmp_dir}"
- makepkg --printsrcinfo
- cd ..
- } |
- if [ "${package%-i18n}-i18n" = "${package}" ]; then
- sed '
- 1 a \\tdepends = '"${package%-i18n}"'
- '
- else
- cat
- fi > \
- "${output}"
-
- )
-
-}
-
-# recursively_umount_and_rm $dir
-# umount all mountpoints in $dir which are also in $dir's
-# filesystem, possibly also $dir itself and then
-# rm -rf --one-file-system $dir
-
-recursively_umount_and_rm() {
- local dir="$1"
-
- if [ -z "${dir}" ]; then
- >&2 echo 'ERROR: recursively_umount_and_rm requires an argument'
- exit 42
- fi
-
- find "${dir}" \
- -xdev -depth -type d \
- -exec 'mountpoint' '-q' '{}' ';' \
- -exec 'sudo' 'umount' '-l' '{}' ';'
- rm -rf --one-file-system "${dir}"
-}
-
-# mangle_pkgbuild $PKGBUILD [$sub_pkgrel]
-# mangle $arch in PKBUILDs to contain i486, i586, i686
-# append $sub_pkgrel to the pkgrel
-
-mangle_pkgbuild() {
- local PKGBUILD="$1"
- local sub_pkgrel="$2"
-
- if [ -n "${sub_pkgrel}" ]; then
- sub_pkgrel=".${sub_pkgrel}"
- fi
-
- sed -i '
- /^arch=[^#]*any/!{
- /^arch=(/s/(/(i686 /
- }
- s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/
- ' "${PKGBUILD}"
-}
-
-# find_newest_of_git_revisions
-# find newest git revision of the ones provided at stdin
-# (assuming linear history)
-
-find_newest_of_git_revisions() {
- local revisions
- local repo
- revisions=$(cat)
-
- if [ "$(
- echo "${revisions}" | \
- wc -l
- )" -eq 1 ]; then
-
- echo "${revisions}"
- return
-
- fi
-
- repo=$(
- find_repository_with_commit \
- "$(
- echo "${revisions}" | \
- grep -xm1 '[0-9a-f]\{40\}'
- )"
- )
-
- eval 'repo="${repo_paths__'"${repo}"'}"'
-
- echo "${revisions}" | \
- xargs -rn1 git -C "${repo}" rev-parse | \
- {
- newest=''
- while read -r current; do
- if [ -z "${newest}" ] || \
- git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then
- newest="${current}"
- fi
- done
- echo "${newest}"
- }
-}
-
-# find_package_repository_to_package $package $git_repository $git_commit
-# find the package repository a package from a given git repository
-# belongs to
-
-find_package_repository_to_package() {
-
- local package="$1"
- local git_repository="$2"
- local git_commit="$3"
- local repo_path
- local repo
-
- eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
- if [ "${git_repository}" = 'archlinux32' ]; then
- repo=$(
- git -C "${repo_path}" archive "${git_commit}" -- | \
- tar -t --wildcards "*/${package}/" | \
- cut -d/ -f1 | \
- sort -u
- )
- else
- repo=$(
- git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \
- tar -t | \
- cut -d/ -f3 | \
- grep -vxF '' | \
- grep -v 'staging\|testing\|-unstable' | \
- grep -v -- '-i686$' | \
- sed 's|-[^-]\+$||' | \
- sort -u
- )
- fi
-
- if [ -z "${repo}" ]; then
- return 1
- fi
-
- if [ "$(
- echo "${repo}" | \
- wc -l
- )" -ne 1 ]; then
- return 1
- fi
-
- echo "${repo}"
-
-}
-
-# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel
-# extract files found in the svn/git source directories
-# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly
-
-extract_source_directory() {
-
- local git_repo="$1"
- local rev="$2"
- local mod_rev="$3"
- local output="$4"
- local sub_pkgrel="$5"
-
- if [ -n "${PKGBUILD}" ]; then
- eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \
- tar -x --strip-components=3 -C "${output}"
- fi
-
- if [ -n "${PKGBUILD_mod}" ]; then
- git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \
- tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \
- true
- git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \
- tar -Ox "${PKGBUILD_mod}" >> \
- "${output}/PKGBUILD"
- fi
-
- mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}"
-
- # shellcheck disable=SC2016
- sed -i '/^\$Id\$$/d' "${output}/PKGBUILD"
-
-}
-
-# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository
-# return a list of packages on the build list which are (run- / build- / check-time)
-# dependencies of the given package
-
-find_dependencies_on_build_list() {
-
- local package="$1"
- local git_revision="$2"
- local mod_git_revision="$3"
- local repository="$4"
-
- generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
- {
- cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
- awk '{print $1 "." $2 "." $3 "." $4}' < \
- "${work_dir}/build-list" | \
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|\.builds|
- ' | \
- xargs -r cat | \
- sort -u
- } | \
- sort | \
- uniq -d
-
-}
-
-# download_sources_by_hash $package $repository $git_revision $git_mod_revision
-# try to download all sources by their hash into the current directory
-# returns 0 if any source was downloaded and 1 otherwise
-
-download_sources_by_hash() {
-
- local package="$1"
- local repository="$2"
- local git_revision="$3"
- local git_mod_revision="$4"
-
- local return_value=1
- local tmp_dir
- local sum_type
- local arch_suffix
-
- tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir)
-
- if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then
- >&2 echo 'download_sources_by_hash: make_source_info failed.'
- rm -rf --one-file-system "${tmp_dir}"
- return 1
- fi
-
- if ! [ -s "${tmp_dir}/.SRCINFO" ]; then
- >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.'
- rm -rf --one-file-system "${tmp_dir}"
- return 1
- fi
-
- for arch_suffix in '' '_i686'; do
- for sum_type in 'sha256sum' 'sha512sum'; do
- grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
- sed 's|^.* = ||' | \
- cat -n > \
- "${tmp_dir}/sums"
- grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
- sed '
- s|^.* = ||
- s|::.*$||
- s|.*/||
- ' | \
- cat -n > \
- "${tmp_dir}/urls"
- if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then
- join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \
- "${tmp_dir}/joined"
- while read -r sum file; do
- if [ "${sum}" = 'SKIP' ]; then
- continue
- fi
- if echo "${sum} ${file}" | \
- ${sum_type} -c > /dev/null 2>&1; then
- # the correct source is already there
- continue
- fi
- if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then
- mv "${tmp_dir}/transfer" "${file}"
- return_value=0
- fi
- done < \
- "${tmp_dir}/joined"
- fi
- done
- done
-
- rm -rf --one-file-system "${tmp_dir}"
- return ${return_value}
-
-}
-
-# expand_version $column_num
-# add "0:" to version in $colum_num-th column if no ":" is there (epoch)
-# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash)
-
-expand_version() {
- local column_num
- column_num="$1"
-
- sed '
- /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/
- /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/
- '
-}
-
-# shrink_version $column_num
-# remove "0:" from version in $colum_num-th column (epoch)
-# remove "+0" from version in $colum_num-th column (git count/hash)
-
-shrink_version() {
- local column_num
- column_num="$1"
-
- sed '
- s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/
- s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/
- '
-}
-
-# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ]
-
-# Return (to stdout) the biggest subset A of the packages in $omega whose
-# runtime dependencies in $omega \cup $keep are also in A
-
-# $all_builds and $all_depends either point to an empty file - then they will get
-# filled with cached data for subsequent calls - or to the same files of a previous
-# call
-
-# If non-empty, $force contains packages which are assumed to match the above
-# condition without checking.
-
-# The arguments are names of files with one $package.$revision.$mod_revision.$repository
-# per line.
-
-find_biggest_subset_of_packages() {
-
- ( # the new shell is intentional
- omega="$1"
- keep="$2"
- all_builds="$3"
- all_depends="$4"
- if [ $# -eq 4 ]; then
- force='/dev/null'
- elif [ $# -eq 5 ]; then
- force="$5"
- else
- >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#"
- return 2
- fi
-
- if [ ! -s "${all_builds}" ]; then
- find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \
- -exec sed '
- s|^|{} |
- s|^\S\+/||
- s|\.builds | |
- ' {} \; | \
- sort -k2,2 > \
- "${all_builds}"
- fi
-
- if [ ! -s "${all_depends}" ]; then
- find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \
- -exec sed '
- s|^|{} |
- s|^\S\+/||
- s|\.run-depends | |
- ' {} \; | \
- grep -v ' base$' | \
- sort -k2,2 > \
- "${all_depends}"
- fi
-
- sort -u "${omega}" | \
- sponge "${omega}"
-
- temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
- {
- sort -u "${keep}"
- cat "${force}" "${force}"
- } | \
- sort | \
- uniq -u > \
- "${temp_dir}/keep.new"
- touch "${temp_dir}/keep"
-
- while [ -s "${temp_dir}/keep.new" ]; do
- cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \
- sort -u | \
- sponge "${temp_dir}/keep"
-
- {
- # we append all packages which are run-dependencies of keep-packages
- # to the keep-list
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.run-depends|
- ' "${temp_dir}/keep" | \
- xargs -r grep -HF '' | \
- sed '
- s|^.*/||
- s|\.run-depends:| |
- ' | \
- sort -u | \
- sort -k2,2 | \
- uniq -f1 | \
- join -1 2 -2 2 -o 2.1 - "${all_builds}"
-
- # we append all packages with run-dependencies on the keep-list
- # to the keep-list
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.builds|
- ' "${temp_dir}/keep" | \
- xargs -r grep -HF '' | \
- sed '
- s|^.*/||
- s|\.builds:| |
- ' | \
- sort -u | \
- sort -k2,2 | \
- uniq -f1 | \
- join -1 2 -2 2 -o 2.1 - "${all_depends}"
- } | \
- sort -u | \
- join -1 1 -2 1 -o 2.1 - "${omega}" | \
- sort -u > \
- "${temp_dir}/keep.new"
-
- # "new" is only what has not been there before and what is not forced
- cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \
- sort | \
- uniq -u | \
- sponge "${temp_dir}/keep.new"
- done
-
- cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \
- sort | \
- uniq -u
-
- )
-
-}
-
-# sort_square_bracket_content $file
-# sort the content of [] in $file, print to stdout
-
-sort_square_bracket_content() {
- local file
- local line
- local token
- local token_list
- local rest
- file="$1"
-
- while read -r line; do
- printf '%s ' "${line}" | \
- tr ' ' '\n' | \
- while read -r token; do
- if echo "${token}" | \
- grep -qF '['; then
- printf '%s[' "${token%[*}"
- token="${token##*[}"
- token_list="${token%,}"
- while ! echo "${token_list}" | \
- grep -qF ']'; do
- read -r token
- token_list=$(
- printf '%s\n' \
- "${token_list}" \
- "${token%,}"
- )
- done
- rest="]${token_list#*]}"
- token_list="${token_list%%]*}"
- token=$(
- printf '%s' "${token_list}" | \
- sort | \
- sed 's|$|,|'
- printf '%s' "${rest}"
- )
- fi
- printf '%s\n' "${token}"
- done | \
- tr '\n' ' ' | \
- sed '
- s|, ]|]|g
- s| $||
- '
- printf '\n'
- done < \
- "${file}"
-}
-
-# smoothen_namcap_log $file
-# remove unneccesary differences from namcap-logs:
-# - remove architecture specific information
-# - sort lines
-# - sort content of square brackets
-
-smoothen_namcap_log() {
- local file
- file="$1"
- # shellcheck disable=SC2016
- sort_square_bracket_content "${file}" | \
- sed '
- # normalize architecture specific information
- s|i[34567]86|$ARCH|g
- s|x86\([-_]64\)\?|$ARCH|g
- # remove haskell hashes
- s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g
- ' | \
- sort | \
- sponge "${file}"
-}
-
-# print_list_of_archaic_packages $source1 $source2 ...
-# print a list of packages which have not been touched for a while,
-# but which are still in the pipeline, e.g. in $source1, $source2 or ...
-
-print_list_of_archaic_packages() {
- for source in "$@"; do
- case "${source}" in
- 'testing')
- # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \
- -exec head -n1 {} \; | \
- "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin
- # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated
- # and no longer block other packages from being moved
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \
- sed '
- s|\.testing$||
- '
- ;;
- 'build-list')
- while read -r pkg rev mod_rev repo; do
- git_repo=$(
- find_repository_with_commit "${rev}"
- )
- eval repo_path='"${repo_paths__'"${git_repo}"'}"'
- commit_date=$(
- git -C "${repo_path}" show -s --format=%ct "${rev}"
- )
- mod_commit_date=$(
- git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}"
- )
- if [ "${mod_commit_date}" -gt "${commit_date}" ]; then
- commit_date="${mod_commit_date}"
- fi
- # packages remaining longer than $max_package_age_build_list days on the build list
- if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then
- printf '%s %s %s %s\n' \
- "${pkg}" \
- "${rev}" \
- "${mod_rev}" \
- "${repo}"
- fi
- done < \
- "${work_dir}/build-list"
- ;;
- 'staging')
- # packages remaining longer than $max_package_age_staging days in staging
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \
- sed '
- s|\.done$||
- '
- ;;
- *)
- >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}"
- ;;
- esac
- done | \
- sort -u
-}
-
-# modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
-# print the given modification revision possibly with a html link to github
-
-modification_revision_link() {
- local mod_rev="$1"
- local repo="$2"
- local pkg="$3"
-
- if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then
- printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \
- "${mod_rev}" \
- "${repo}" \
- "${pkg}" \
- "${mod_rev}"
- else
- printf '%s\n' \
- "${mod_rev}"
- fi
-}
-
-# trigger_mirror_refreshs
-# trigger a refresh of capable tier 1 mirrors (as backup for master mirror)
-
-trigger_mirror_refreshs() {
- local tmp_file
-
- tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir)
- date '+%s' > \
- "${tmp_file}"
- ${master_mirror_rsync_command} \
- "${tmp_file}" \
- "${master_mirror_rsync_directory}/lastupdate"
- rm "${tmp_file}"
- for trigger_url in ${mirror_refresh_trigger_urls}; do
- screen -S trigger-mirror-update -d -m curl -L "${trigger_url}"
- done
-}
-
-# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name
-extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() {
- pkgname="$1"
- pkgname="${pkgname%.pkg.tar.xz}"
- arch="${pkgname##*-}"
- pkgname="${pkgname%-*}"
- sub_pkgrel="${pkgname##*-}"
- pkgname="${pkgname%-*}"
- pkgrel="${sub_pkgrel%.*}"
- if [ "${pkgrel}" = "${sub_pkgrel}" ]; then
- sub_pkgrel='0'
- else
- sub_pkgrel="${sub_pkgrel##*.}"
- fi
- epoch="${pkgname##*-}"
- pkgname="${pkgname%-*}"
- pkgver="${epoch#*:}"
- if [ "${pkgver}" = "${epoch}" ]; then
- epoch='0'
- else
- epoch="${epoch%%:*}"
- fi
-}
diff --git a/bin/mysql-functions b/bin/mysql-functions
deleted file mode 100755
index 185d9a9..0000000
--- a/bin/mysql-functions
+++ /dev/null
@@ -1,1098 +0,0 @@
-#!/bin/sh
-
-# contains functions used to access mysql db
-
-# shellcheck disable=SC2016,SC2039
-
-if [ -z "${base_dir}" ]; then
- # just to make shellcheck happy
- . 'conf/default.conf'
-fi
-
-# base64_encode_each encode each line of stdin with base64
-
-base64_encode_each() {
- local line
-
- while read -r line; do
- printf '%s' \
- "${line}" | \
- base64 -w0
- printf '\n'
- done
-}
-
-# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
-
-# shellcheck disable=SC2086
-mysql_add_package_source() {
- local names='pkgbase git_revision mod_git_revision upstream_package_repository'
- local values
- local uses_upstream
- local uses_modification
- local repo
-
- if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then
- uses_modification=1
- else
- uses_modification=0
- fi
- uses_upstream=0
- for repo in ${repo_names}; do
- if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \
- tar -t 2>/dev/null | \
- sed 's,-x86_64/,-any/,' | \
- grep -qFx "$1/repos/$4-any/PKGBUILD"; then
- uses_upstream=1
- fi
- done
-
- for _ in ${names}; do
- values="${values}$(
- printf '%s' "$1" | \
- base64 -w0
- ) "
- shift
- done
- values="${values% }"
-
- {
- printf 'INSERT IGNORE INTO package_sources'
- printf ' ('
- printf '`%s`, ' ${names}
- printf '`uses_upstream`,`uses_modification`'
- printf ') SELECT'
- printf ' from_base64("%s"), ' ${values% *}
- printf ' `upstream_repositories`.`id`,%s,%s' \
- ${uses_upstream} ${uses_modification}
- printf ' FROM `upstream_repositories`'
- printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \
- "${values##* }"
- } | \
- ${mysql_command}
-}
-
-# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository
-
-# shellcheck disable=SC2031,SC2086,SC2154
-mysql_add_binary_package() {
- local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository'
- local name
- for name in ${names}; do
- eval 'local '"${name}"
- eval "${name}"'=$(
- printf "%s" "$1" |
- base64 -w0
- )'
- shift
- done
-
- {
- printf 'INSERT IGNORE INTO binary_packages'
- printf ' ('
- printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted'
- printf ') SELECT'
- printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}"
- printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures'
- printf ' 0, 0, 0'
- printf ' FROM'
- printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures'
- printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
- printf ' WHERE'
- printf ' `%s`.`name` = from_base64("%s") AND' \
- 'repositories' "${repository}" \
- 'architectures' "${architecture}"
- printf ' `package_sources`.`%s` = from_base64("%s") AND' \
- 'pkgbase' "${pkgbase}" \
- 'git_revision' "${git_revision}" \
- 'mod_git_revision' "${mod_git_revision}"
- printf ' `upstream_repositories`.`name` = from_base64("%s")' \
- "${upstream_package_repository}"
- } | \
- sed '
- s|, )|)|g
- s|, FROM| FROM|g
- ' | \
- ${mysql_command}
-}
-
-# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel
-
-# shellcheck disable=SC2031,SC2086,SC2154
-mysql_show_binary_package() {
- local names='pkgname pkgver pkgrel sub_pkgrel'
- local name
- for name in ${names}; do
- eval 'local '"${name}"
- eval "${name}"'=$(
- printf "%s" "$1" |
- base64 -w0
- )'
- shift
- done
-
- {
- printf 'SELECT'
- printf ' `%s`.`%s`,' \
- 'repositories' 'name' \
- 'binary_packages' 'pkgname' \
- 'package_sources' 'pkgver' \
- 'package_sources' 'pkgrel' \
- 'binary_packages' 'sub_pkgrel' \
- 'architectures' 'name' \
- 'package_sources' 'pkgbase' \
- 'package_sources' 'git_revision' \
- 'package_sources' 'mod_git_revision' \
- 'upstream_repositories' 'name'
- printf ' FROM `binary_packages`'
- mysql_join_binary_packages_architectures
- mysql_join_binary_packages_repositories
- mysql_join_binary_packages_build_assignments
- mysql_join_build_assignments_package_sources
- mysql_join_package_sources_upstream_repositories
- printf ' WHERE'
- printf ' `%s`.`%s` = from_base64("%s") AND' \
- 'binary_packages' 'pkgname' "${pkgname}" \
- 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \
- 'package_sources' 'pkgver' "${pkgver}" \
- 'package_sources' 'pkgrel' "${pkgrel}"
- printf ';'
- } | \
- sed '
- s|, FROM| FROM|g
- s|AND;|;|g
- ' | \
- ${mysql_command} --html
-}
-
-# mysql_add_install_target $install_target
-
-# shellcheck disable=2086
-mysql_add_install_target() {
- local install_target
- install_target=$(
- printf "%s" "$1" | \
- base64 -w0
- )
-
- {
- printf 'INSERT IGNORE INTO `install_targets` (`name`)'
- printf ' VALUES (from_base64("%s"))' \
- "${install_target}"
- } | \
- ${mysql_command}
-}
-
-# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository
-# or
-# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository
-# if sub_pkgrel should be determined automatically
-# and
-# mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository
-# or
-# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository
-# if $sub_pkgrel should be forced
-
-# generate the meta data of a package (dependencies, built packages, ...) in the database
-
-mysql_generate_package_metadata() {
-
- ( # new shell is intentional
- case "$1" in
- ''|*[!0-9]*)
- unset forced_sub_pkgrel
- ;;
- *)
- forced_sub_pkgrel=$(
- printf '%s' "$1" | \
- base64 -w0
- )
- shift
- ;;
- esac
- current_repository="$1"
- package="$2"
-
- if [ $# -eq 2 ]; then
- # second form
- repository="${package##*.}"
- package="${package%.*}"
- mod_git_revision="${package##*.}"
- package="${package%.*}"
- git_revision="${package##*.}"
- package="${package%.*}"
- else
- git_revision="$3"
- mod_git_revision="$4"
- repository="$5"
- fi
-
- temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
- printf '.' >&2
- if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then
- printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"
- exit 2
- fi
- # remove empty lines and unsupported architectures
- sed -i '
- /^[^=]*=\s*$/d
- /^\s*arch = /{
- / \(i686\|any\)$/!d
- }
- ' "${temp_dir}/SRCINFO"
-
- if [ ! -s "${temp_dir}/SRCINFO" ]; then
- >&2 printf '"make_source_info" had empty output - eh, what?\n'
- exit 2
- fi
- printf '\n\n' >> "${temp_dir}/SRCINFO"
-
- printf '.' >&2
- pkgbase=$(
- grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \
- cut -d' ' -f3
- )
- if [ -z "${pkgbase}" ]; then
- >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n'
- exit 2
- fi
-
- # add the package source
- mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
- printf '.' >&2
-
- # now we encode everything in base64
- current_repository=$(
- printf '%s' "${current_repository}" | \
- base64 -w0
- )
- pkgbase=$(
- printf '%s' "${pkgbase}" | \
- base64 -w0
- )
- git_revision=$(
- printf '%s' "${git_revision}" | \
- base64 -w0
- )
- mod_git_revision=$(
- printf '%s' "${mod_git_revision}" | \
- base64 -w0
- )
- repository=$(
- printf '%s' "${repository}" | \
- base64 -w0
- )
-
- # add the build assignment(s)
- {
- archs=$(
- sed -n '
- s/^\tarch = //
- T
- p
- ' "${temp_dir}/SRCINFO" | \
- grep -vxF 'any' | \
- sort -u
- )
- if [ -z "${archs}" ]; then
- echo 'any'
- else
- printf '%s\n' "${archs}"
- fi
- } | \
- while read -r arch; do
- printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
- printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0'
- printf ' FROM `architectures` JOIN `package_sources`'
- printf ' WHERE `architectures`.`name` = from_base64("%s")' \
- "$(
- printf '%s' "${arch}" | \
- base64 -w0
- )"
- printf ' AND `package_sources`.`%s` = from_base64("%s")' \
- 'pkgbase' "${pkgbase}" \
- 'git_revision' "${git_revision}" \
- 'mod_git_revision' "${mod_git_revision}"
- printf ';\n'
- done > \
- "${temp_dir}/add-build-assignments-command"
-
- # TODO: correctly link between binary_packages and build_assignments using any_arch
-
- # shellcheck disable=SC2034
- # select any specific arch (which will be building the 'any' part of a split package)
- any_arch=$(
- {
- sed -n '
- s/^\tarch = //
- T
- p
- ' "${temp_dir}/SRCINFO" | \
- sort -r | \
- grep -vxFm 1 'any' || \
- echo 'any'
- } | \
- base64_encode_each
- )
-
- grep '^pkgname = ' "${temp_dir}/SRCINFO" | \
- cut -d' ' -f3 | \
- while read -r pkgname; do
- pkgname64=$(
- printf '%s' "${pkgname}" | \
- base64 -w0
- )
- sed -n '
- /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{
- :a
- N
- /\n$/{
- p
- T
- }
- ba
- }
- ' "${temp_dir}/SRCINFO" | \
- sed '
- /^\S/d
- s/^\s*//
- ' > \
- "${temp_dir}/BINARYINFO.${pkgname64}"
-
- grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \
- cut -d' ' -f3 | \
- while read -r arch; do
- arch64=$(
- printf '%s' "${arch}" | \
- base64 -w0
- )
- sed '
- s/^\(\S\+\)_'"${arch}"' = /\1 = /
- ' "${temp_dir}/BINARYINFO.${pkgname64}" > \
- "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}"
- done
- done
- find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \
- while read -r _ pkgname arch; do
- pkgver=$(
- grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- base64_encode_each
- )
- pkgrel=$(
- grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- base64_encode_each
- )
- epoch=$(
- {
- grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \
- echo 'epoch = 0'
- } | \
- cut -d' ' -f3 | \
- base64_encode_each
- )
- provides=$(
- grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- makedepends=$(
- grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- checkdepends=$(
- grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- rundepends=$(
- grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- if [ -n "${forced_sub_pkgrel}" ]; then
- sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")'
- else
- sub_pkgrel=$(
- printf '(SELECT COALESCE('
- # do not add binary packages which are currently on the
- # build-list or in $current_repository (beware of split
- # packages!)
- printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
- mysql_join_binary_packages_architectures
- mysql_join_binary_packages_repositories
- printf ' WHERE'
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- printf ' `architectures`.`name`=from_base64("%s")' \
- "${arch}"
- printf ' AND `repositories`.`name` IN ("build-list",from_base64("%s"))),' \
- "${current_repository}"
- # max(sub_pkgrel)+1
- printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
- mysql_join_binary_packages_architectures
- printf ' WHERE'
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then
- # 'any' gets higher sub_pkgrel than any architecture
- printf ' 1'
- else
- # not-'any' gets higher sub_pkgrel than same or 'any' architecture
- printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \
- "${arch}"
- fi
- printf ')'
- printf ',0))'
- )
- fi
- {
- printf 'INSERT IGNORE INTO `binary_packages` ('
- printf '`%s`,' \
- 'build_assignment' \
- 'repository' \
- 'architecture' \
- 'epoch' \
- 'pkgver' \
- 'pkgrel' \
- 'pkgname' \
- 'sub_pkgrel' \
- 'has_issues' \
- 'is_tested' \
- 'is_to_be_deleted'
- printf ') SELECT '
- printf '`%s`.`id`,' \
- 'build_assignments' \
- 'repositories' \
- 'architectures'
- printf 'from_base64("%s"),' \
- "${epoch}" \
- "${pkgver}" \
- "${pkgrel}" \
- "${pkgname}"
- printf '%s,0,0,0 FROM' \
- "${sub_pkgrel}"
- printf ' `%s` JOIN' \
- 'repositories' \
- 'architectures' \
- 'build_assignments'
- mysql_join_build_assignments_package_sources
- mysql_join_package_sources_upstream_repositories
- printf ' WHERE'
- printf ' `%s`.`%s` = from_base64("%s") AND' \
- 'repositories' 'name' "${current_repository}" \
- 'architectures' 'name' "${arch}" \
- 'package_sources' 'pkgbase' "${pkgbase}" \
- 'package_sources' 'git_revision' "${git_revision}" \
- 'package_sources' 'mod_git_revision' "${mod_git_revision}" \
- 'upstream_repositories' 'name' "${repository}"
- printf ';\n'
- } | \
- sed '
- s|,)|)|g
- s| JOIN JOIN | JOIN |
- s| AND;$|;|
- ' >> \
- "${temp_dir}/add-binary-packages-command"
- {
- printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \
- 'provides' \
- 'makedepends' \
- 'checkdepends' \
- 'rundepends'
-
- printf 'INSERT INTO `provides` VALUES\n'
- echo "${provides}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' (from_base64("%s"));\n' \
- "${pkgname}"
-
- printf 'INSERT INTO `rundepends` VALUES\n'
- echo "${rundepends}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' ("base");\n'
-
- echo "${checkdepends}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- 1 s/^/INSERT INTO `checkdepends` VALUES \n/
- $ s/,$/;/
- '
-
- printf 'INSERT INTO `makedepends` VALUES\n'
- echo "${makedepends}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' ("base-devel");\n'
-
- printf 'INSERT IGNORE INTO `install_targets` (`name`)'
- printf ' SELECT (`name`) FROM `%s` UNION' \
- 'provides' \
- 'makedepends' \
- 'checkdepends' \
- 'rundepends' | \
- sed 's| UNION$|;\n|'
-
- for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
- case "${link}" in
- 'provides')
- printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT'
- printf ' `binary_packages`.`id`,`install_targets`.`id` FROM'
- ;;
- 'makedepends'|'checkdepends'|'rundepends')
- printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT'
- printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM'
- printf ' `dependency_types` JOIN'
- ;;
- esac
- printf ' `binary_packages`'
- mysql_join_binary_packages_architectures
- printf ' JOIN `install_targets`'
- printf ' JOIN `%s`' "${link}"
- printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
- printf ' WHERE'
- if [ "${link}" = 'makedepends' ] || \
- [ "${link}" = 'checkdepends' ] || \
- [ "${link}" = 'rundepends' ]; then
- printf ' `dependency_types`.`name` = "%s" AND' \
- "${link%depends}"
- fi
- printf ' `binary_packages`.`%s` = from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- # we do not want to match the sub_pkgrel:
- # a) it is tedious to do so (because it may be calculated
- # dynamically)
- # b) it is not necessary to do so: if only the sub_pkgrel
- # changed, the dependencies and provided install_targets
- # should not have changed
- printf ' `architectures`.`name` = from_base64("%s");\n' \
- "${arch}"
- # the repository is of no relevance: it hardly matters for
- # the dependencies
- done
-
- printf 'DROP TABLE `%s`;\n' \
- 'provides' \
- 'makedepends' \
- 'checkdepends' \
- 'rundepends'
- } >> \
- "${temp_dir}/add-install-targets-command"
- done
- printf '.' >&2
-
- {
- if [ -s "${temp_dir}/add-build-assignments-command" ]; then
- cat "${temp_dir}/add-build-assignments-command"
- fi
- if [ -s "${temp_dir}/add-binary-packages-command" ]; then
- cat "${temp_dir}/add-binary-packages-command"
- fi
- if [ -s "${temp_dir}/add-install-targets-command" ]; then
- cat "${temp_dir}/add-install-targets-command"
- fi
- } | \
- ${mysql_command}
- printf '.' >&2
-
- )
-}
-
-# mysql_sanity_check
-# do a sanity check on the mysql database
-
-mysql_sanity_check() {
- {
- printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)'
- printf ' FROM `binary_packages`'
- mysql_join_binary_packages_build_assignments
- mysql_join_binary_packages_architectures '' 'bp_arch'
- mysql_join_build_assignments_architectures '' 'ba_arch'
- printf ' WHERE `bp_arch`.`name`!="any"'
- printf ' AND `ba_arch`.`name`="any";\n'
- printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)'
- printf ' FROM `binary_packages` AS `a`'
- printf ' JOIN `binary_packages` AS `b`'
- printf ' ON `a`.`pkgname`=`b`.`pkgname`'
- printf ' AND `a`.`repository`=`b`.`repository`'
- printf ' AND `a`.`id`!=`b`.`id`'
- mysql_join_binary_packages_repositories 'a'
- printf ' WHERE `repositories`.`name`="build-list";\n'
- printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)'
- printf ' FROM `binary_packages` AS `a`'
- printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`'
- mysql_join_binary_packages_repositories 'a' 'arep'
- mysql_join_binary_packages_repositories 'b' 'brep'
- printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`'
- printf ' AND `%srep`.`name`="build-list"' \
- 'a' 'b'
- printf ';\n'
- } | \
- ${mysql_command} --raw --batch | \
- sed '
- /^CONCAT("/d
- s,^,<font color="#FF0000">,
- s,$,</font>,
- '
- ( # new shell is intentional
- temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
- for dir in $(ls_master_mirror 'i686'); do
- ls_master_mirror "i686/${dir}" | \
- sed -n '
- s/\.pkg\.tar\.xz$//
- T
- s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/
- s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/
- s|^|'"${dir}"'/|
- p
- '
- done | \
- sort > \
- "${temp_dir}/master-mirror-listing"
-
- {
- printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`'
- printf ' FROM `binary_packages`'
- mysql_join_binary_packages_architectures
- mysql_join_binary_packages_repositories
- printf ' WHERE `repositories`.`is_on_master_mirror`'
- } | \
- ${mysql_command} --batch | \
- sed '
- 1d
- s,\t,/,
- s,\t,-,
- s,\t,:,
- s,\t,-,
- s,\t,.,
- s,\t,-,
- ' | \
- sort > \
- "${temp_dir}/mysql-packages"
-
- diff -u \
- "${temp_dir}/master-mirror-listing" \
- "${temp_dir}/mysql-packages"
- )
-}
-
-mysql_find_build_assignment_loops() {
- new_loops=$(
- {
- printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`'
- printf ' FROM `dependencies`'
- mysql_join_dependencies_install_target_providers
- mysql_join_install_target_providers_binary_packages '' 'packages_dependency'
- mysql_join_dependencies_binary_packages '' 'packages_dependent'
- mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency'
- mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent'
- printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"'
- } | \
- ${mysql_command} --raw --batch | \
- sed '
- 1d
- y/\t/ /
- ' | \
- tsort 2>&1 >/dev/null | \
- sed 's/^tsort:\s*//' | \
- {
- loop=0
- while read -r id; do
- if [ "x${id}" = 'x-: input contains a loop:' ]; then
- loop=$((loop+1))
- continue
- fi
- if ! printf '%s' "${id}" | tr '\n' ' ' | grep -q '^[0-9]\+$'; then
- >&2 printf 'ERROR: non-numeric id "%s"\n' "${id}"
- continue
- fi
- printf '(%s,%s),' "${loop}" "${id}"
- done | \
- sed 's/,$//'
- }
- )
- {
- printf 'DELETE FROM `build_dependency_loops`;\n'
- if [ -n "${new_loops}" ]; then
- printf 'INSERT INTO `build_dependency_loops` (`loop`,`build_assignment`) VALUES %s;\n' \
- "${new_loops}"
- fi
- } | \
- ${mysql_command}
-}
-
-# mysql_cleanup [dry]
-# clean up left overs from mysql database
-mysql_cleanup() {
- local operator
- if [ "$#" = '0' ]; then
- operator='DELETE'
- elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then
- operator='SELECT COUNT(*)'
- else
- >&2 echo 'Unknown parameter'
- >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".'
- exit 2
- fi
- {
- # remove to-be-decided binary_packages
- printf '%s ' \
- "${operator}"
- if [ "${operator}" = 'DELETE' ]; then
- printf '`binary_packages` '
- fi
- printf 'FROM `binary_packages`'
- mysql_join_binary_packages_repositories
- printf ' WHERE `repositories`.`name`="to-be-decided";\n'
- # remove dependencies w/o binary_package or install_target
- printf '%s FROM `dependencies` ' \
- "${operator}"
- printf 'WHERE NOT EXISTS '
- printf '('
- printf 'SELECT * FROM `binary_packages` '
- printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`'
- printf ') OR NOT EXISTS '
- printf '('
- printf 'SELECT * FROM `install_targets` '
- printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`'
- printf ');\n'
- # remove install_target_providers w/o binary_package or install_target
- printf '%s FROM `install_target_providers` ' \
- "${operator}"
- printf 'WHERE NOT EXISTS '
- printf '('
- printf 'SELECT * FROM `binary_packages` '
- printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`'
- printf ') OR NOT EXISTS '
- printf '('
- printf 'SELECT * FROM `install_targets` '
- printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`'
- printf ');\n'
- # remove build_assignments w/o binary_package
- printf '%s FROM `build_assignments` ' \
- "${operator}"
- printf 'WHERE NOT EXISTS '
- printf '('
- printf 'SELECT * FROM `binary_packages` '
- printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`'
- printf ');\n'
- # remove failed_builds with unbroken build_assignments
- printf '%s ' \
- "${operator}"
- if [ "${operator}" = 'DELETE' ]; then
- printf '`failed_builds` '
- fi
- printf 'FROM `failed_builds` '
- mysql_join_failed_builds_build_assignments
- printf 'WHERE NOT `build_assignments`.`is_broken`'
- printf ';\n'
- # remove failed_builds w/o build_assignment
- printf '%s FROM `failed_builds` ' \
- "${operator}"
- printf 'WHERE NOT EXISTS '
- printf '('
- printf 'SELECT * FROM `build_assignments` '
- printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`'
- printf ');\n'
- # remove package_sources w/o build_assignment
- printf '%s FROM `package_sources` ' \
- "${operator}"
- printf 'WHERE NOT EXISTS '
- printf '('
- printf 'SELECT * FROM `build_assignments` '
- printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`'
- printf ');\n'
- printf 'UPDATE `build_slaves`'
- mysql_join_build_slaves_binary_packages
- mysql_join_binary_packages_repositories
- printf ' SET `currently_building`=NULL'
- printf ' WHERE `repositories`.`name`!="build-list";\n'
- } | \
- ${mysql_command}
-}
-
-# mysql_query_has_pending_dependencies `build_assignment`.`id`
-# print a mysql query giving wether dependencies are pending
-mysql_query_has_pending_dependencies() {
- printf 'EXISTS ('
- printf 'SELECT * FROM `binary_packages` as `to_dos`'
- mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos'
- mysql_join_binary_packages_dependencies 'to_dos'
- mysql_join_dependencies_install_target_providers
- mysql_join_install_target_providers_binary_packages '' 'bin_deps'
- mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos'
- printf ' WHERE'
- printf ' `%s`.`name`="build-list" AND' \
- 'dep_repos' 'to_do_repos'
- printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND'
- printf ' `to_dos`.`build_assignment`=%s' \
- "$1"
- printf ')'
-}
-
-# mysql_query_is_part_of_loop `build_assignment`.`id`
-# print a mysql query giving wether the package is part of a loop
-mysql_query_is_part_of_loop() {
- printf 'EXISTS ('
- printf 'SELECT * FROM `build_dependency_loops`'
- printf ' WHERE `build_dependency_loops`.`build_assignment`=%s' \
- "$1"
- printf ')'
-}
-
-# mysql_query_select_pkgbase_and_revision
-# print the part of a mysql query giving:
-# pkgbase git_revision mod_git_revision upstream_package_repository
-mysql_query_select_pkgbase_and_revision() {
- printf '`package_sources`.`%s`,' \
- 'pkgbase' \
- 'git_revision' \
- 'mod_git_revision'
- printf '`upstream_repositories`.`name`'
- printf ' FROM `build_assignments`'
- mysql_join_build_assignments_package_sources
- mysql_join_package_sources_upstream_repositories
-}
-
-# mysql_repair_binary_packages_without_build_assignment
-# try to generate valid build assignments to binary packages without
-# a valid one yet
-mysql_repair_binary_packages_without_build_assignment() {
- {
- printf 'SELECT '
- printf '`binary_packages`.`id`'
- printf ',replace(to_base64(%s),"\\n","")' \
- '`binary_packages`.`pkgname`' \
- '`architectures`.`name`'
- printf ' FROM `binary_packages`'
- mysql_join_binary_packages_architectures
- printf ' WHERE `binary_packages`.`build_assignment`<0'
- } | \
- ${mysql_command} --raw --batch | \
- sed '1d' | \
- while read -r id pkgname arch; do
- pkgname=$(
- printf '%s' "${pkgname}" | \
- base64 -d
- )
- pkgbase=$(
- curl -Ss "$(
- printf 'https://www.archlinux.org/packages/search/json/?name=%s' \
- "${pkgname}"
- )" | \
- sed '
- s/^.*"results":\s*\[//
- s/}\s*,\s*{/\n/g
- ' | \
- grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \
- tr ',' '\n' | \
- grep '"pkgbase":' | \
- cut -d'"' -f4 | \
- sort -u | \
- head -n1
- )
- if [ -z "${pkgbase}" ] && \
- {
- printf 'SELECT count(*) FROM `package_sources`'
- printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
- "$(printf '%s' "${pkgname}" | base64 -w0)"
- } | \
- ${mysql_command} --raw --batch | \
- sed '1d' | \
- grep -qvxF '0'; then
- pkgbase="${pkgname}"
- fi
- if [ -z "${pkgbase}" ]; then
- >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}"
- continue
- fi
- pkgbase=$(
- printf '%s' "${pkgbase}" | \
- base64 -w0
- )
- printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
- printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0'
- printf ' FROM `package_sources`'
- printf ' JOIN `architectures`'
- printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}"
- printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}"
- printf ' LIMIT 1;\n'
- printf 'UPDATE `binary_packages`'
- printf ' JOIN `build_assignments`'
- mysql_join_binary_packages_build_assignments
- printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`'
- printf ' WHERE `binary_packages`.`id`=%s' "${id}"
- printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}"
- done | \
- ${mysql_command}
-}
-
-# mysql_remove_duplicate_binary_packages
-# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel,
-# having difference of 1 in sub_pkgrel
-
-mysql_remove_duplicate_build_order() {
- {
- printf 'CREATE TEMPORARY TABLE `ren`'
- printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n'
- printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)'
- printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`'
- printf ' FROM `binary_packages` as `old`'
- printf ' JOIN `binary_packages` as `new` ON'
- printf ' `old`.`%s`=`new`.`%s` AND' \
- 'pkgname' 'pkgname' \
- 'epoch' 'epoch' \
- 'pkgver' 'pkgver' \
- 'pkgrel' 'pkgrel'
- printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`'
- mysql_join_binary_packages_repositories 'old' 'orep'
- mysql_join_binary_packages_repositories 'new' 'nrep'
- printf ' WHERE `orep`.`name`!="to-be-decided"'
- printf ' AND `nrep`.`name`="to-be-decided";\n'
- printf 'UPDATE IGNORE `dependencies`'
- printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`'
- printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n'
- printf 'UPDATE IGNORE `install_target_providers`'
- printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`'
- printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n'
- printf 'DELETE FROM `binary_packages`'
- printf ' WHERE EXISTS ('
- printf 'SELECT * FROM `ren`'
- printf ' WHERE `ren`.`old`=`binary_packages`.`id`'
- printf ');\n'
- printf 'UPDATE IGNORE `binary_packages`'
- printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`'
- printf ' SET `binary_packages`.`repository`=`ren`.`repo`,'
- printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n'
- } | \
- ${mysql_command}
-}
-
-# mysql_package_name_query
-# print a mysql query of the full name of a package file
-
-mysql_package_name_query() {
- printf 'CONCAT('
- printf '`binary_packages`.`pkgname`,"-",'
- printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
- printf '`binary_packages`.`pkgver`,"-",'
- printf '`binary_packages`.`pkgrel`,".",'
- printf '`binary_packages`.`sub_pkgrel`,"-",'
- printf '`architectures`.`name`,".pkg.tar.xz"'
- printf ')'
-}
-
-# mysql_join_*_*
-# print 'JOIN' part of mysql query to connect the respective tables
-# these functions take 2 optional arguments, acting as aliases for
-# the tables
-
-# mysql_join__generic $table_a $column_a $table_b $column_b
-# create mysql_join_${table_a}_${table_b}() function
-
-mysql_join__generic() {
- eval "$(
- printf 'mysql_join_%s_%s() {\n' "$1" "$3"
- printf ' printf '"'"' JOIN `%s`'"'"'\n' "$3"
- printf ' if [ -n "$2" ]; then\n'
- printf ' printf '"'"' AS `%%s`'"'"' "$2"\n'
- printf ' fi\n'
- printf ' if [ -n "$1" ]; then\n'
- printf ' printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2"
- printf ' else\n'
- printf ' printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2"
- printf ' fi\n'
- printf ' if [ -n "$2" ]; then\n'
- printf ' printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4"
- printf ' else\n'
- printf ' printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4"
- printf ' fi\n'
- printf '}\n'
- )"
-}
-
-for link in \
- 'binary_packages:architecture:architectures' \
- 'binary_packages:repository:repositories' \
- 'binary_packages:build_assignment:build_assignments' \
- \
- 'build_assignments:architecture:architectures' \
- 'build_assignments:package_source:package_sources' \
- \
- 'build_dependency_loops:build_assignment:build_assignments' \
- 'build_dependency_loops:build_assignment build_assignment:binary_packages' \
- \
- 'build_slaves:currently_building:build_assignments' \
- 'build_slaves:currently_building build_assignment:binary_packages' \
- \
- 'dependencies:depending_on:install_targets' \
- 'dependencies:dependent:binary_packages' \
- 'dependencies:dependency_type:dependency_types' \
- \
- 'failed_builds:reason:fail_reason' \
- 'failed_builds:build_assignment:build_assignments' \
- 'failed_builds:build_slave:build_slaves' \
- \
- 'install_target_providers:package:binary_packages' \
- 'install_target_providers:install_target:install_targets' \
- 'install_target_providers:install_target depending_on:dependencies' \
- \
- 'package_sources:upstream_package_repository:upstream_repositories' \
- \
- 'repositories:stability:repository_stabilities' \
- \
- 'upstream_repositories:git_repository:git_repositories'; do
-# A join for these cannot be done, because it's not clear on what to join:
-# 'repository_stability_relations:more_stable:repository_stabilities'
-# 'repository_stability_relations:less_stable:repository_stabilities'
-
- table_b="${link##*:}"
- table_a="${link%:*}"
- column_b="${table_a##*:}"
- table_a="${table_a%:*}"
- column_a="${column_b% *}"
- if [ "${column_a}" = "${column_b}" ]; then
- column_b='id'
- else
- column_b="${column_b##* }"
- fi
-
- mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}"
- mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}"
-done
generated by cgit v1.2.3-70-g09d2 (git 2.47.0) at 2024-11-25 16:12:18 +0000