summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2018-02-08 14:10:07 +0100
committerErich Eckner <git@eckner.net>2018-02-08 14:10:07 +0100
commit54f1d508044492fdd4410defbc94963bd0608d63 (patch)
tree4ce4cc5aef0f90bb970227809b74db32b34a7eb5
parent5c5b3b2ef97347ba70458ea02989e838b5c76393 (diff)
downloadbuilder-54f1d508044492fdd4410defbc94963bd0608d63.tar.xz
bin/get-package-updates: take dependency information for black listing from database
-rwxr-xr-xbin/get-package-updates226
1 files changed, 87 insertions, 139 deletions
diff --git a/bin/get-package-updates b/bin/get-package-updates
index e57d72b..4fcefd9 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -367,18 +367,21 @@ echo 'apply blacklisting'
black_listed=''
black_listed_new=$(
{
- git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \
- tar -Ox 'blacklist' | \
- sed '
- s/\s*#.*$//
- /^\s*$/d
- '
- if [ -n "${test_exclusion}" ]; then
- echo "${test_exclusion}"
- fi
+ {
+ git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \
+ tar -Ox 'blacklist' | \
+ sed '
+ s/\s*#.*$//
+ /^\s*$/d
+ '
+ if [ -n "${test_exclusion}" ]; then
+ echo "${test_exclusion}"
+ fi
+ } | \
+ base64_encode_each
# shellcheck disable=SC2016
{
- printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
+ printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")'
printf ' FROM `package_sources`'
printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
} | \
@@ -388,120 +391,91 @@ black_listed_new=$(
sort -u
)
-find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' -printf '%f\n' | \
- grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.builds$' | \
- sed '
- s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$|\2 \3 \4 \1|
- ' | \
- sort -u | \
- sort -k4,4 > \
- "${tmp_dir}/packages"
-
-{
- uniq -f3 -u "${tmp_dir}/packages" | \
- awk '{print $4 " " $4 "." $1 "." $2 "." $3}'
- uniq -f3 -D "${tmp_dir}/packages" | \
- uniq -f3 --group=append | \
- while read -r rev mod_rev repo pkg; do
- if [ -z "${rev}" ] && \
- [ -z "${mod_rev}" ] && \
- [ -z "${repo}" ] && \
- [ -z "${pkg}" ]; then
- # shellcheck disable=SC2031
- printf '%s %s.%s.%s.%s\n' \
- "${opkg}" \
- "${opkg}" \
- "$(
- # shellcheck disable=SC2086
- printf '%s\n' ${revs} | \
- sort -u | \
- find_newest_of_git_revisions
- )" \
- "$(
- # shellcheck disable=SC2086
- printf '%s\n' ${mod_revs} | \
- sort -u | \
- find_newest_of_git_revisions
- )" \
- "${orepo}" | \
- grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+$'
- revs=''
- mod_revs=''
- continue
- fi
- revs=$(
- printf '%s\n' \
- ${revs} \
- "${rev}"
- )
- mod_revs=$(
- printf '%s\n' \
- ${mod_revs} \
- "${mod_rev}"
- )
- opkg="${pkg}"
- orepo="${repo}"
- done
-} | \
- sort -k1,1 > \
- "${tmp_dir}/newest-revisions"
-
-sed '
- s|^\S\+ |'"${work_dir}"'/package-infos/|
- s|$|.builds|
-' "${tmp_dir}/newest-revisions" | \
- xargs -r cat | \
- sort | \
- uniq -c > \
- "${work_dir}/built-packages"
-
-sed '
- s|^\S\+ |'"${work_dir}"'/package-infos/|
- s|$|.build-depends|
-' "${tmp_dir}/newest-revisions" | \
- xargs -r grep -HF '' | \
- sed '
- s|\(\.[^.]\+\)\{4\}:\([^:]\+\)$| \2|
- s|^.*/||
- ' | \
- sort -k2,2 > \
- "${work_dir}/newest-dependencies"
-
while [ -n "${black_listed_new}" ]; do
black_listed=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${black_listed} ${black_listed_new} | \
+ printf '%s\n' "${black_listed}" "${black_listed_new}" | \
+ grep -vxF '' | \
sort -u
)
black_listed_new=$(
{
- # shellcheck disable=SC2086
- printf '%s\n' ${black_listed} | \
- sort -k1,1 | \
- join -j1 -o 2.2 - "${tmp_dir}/newest-revisions" | \
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.builds|
- ' | \
- xargs -r cat | \
- sort | \
- uniq -c
- cat "${work_dir}/built-packages"
+ printf '%s\n' "${black_listed}" "${black_listed}"
+
+ # shellcheck disable=SC2016
+ {
+ printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
+ printf 'INSERT INTO `bl` (`pkgbase`) VALUES '
+ printf '(from_base64("%s")),\n' "${black_listed}" | \
+ sort -u | \
+ sed '
+ $ s/,$/;/
+ '
+ printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")'
+ printf ' FROM `package_sources` AS `a_ps`'
+ printf ' JOIN `build_assignments` AS `a_ba` ON `a_ba`.`package_source`=`a_ps`.`id`'
+ printf ' JOIN `binary_packages` AS `a_bp` ON `a_bp`.`build_assignment`=`a_ba`.`id`'
+ printf ' JOIN `dependencies` ON `dependencies`.`dependent`=`a_bp`.`id`'
+ printf ' WHERE NOT EXISTS ('
+ printf ' SELECT *'
+ printf ' FROM `install_target_providers`'
+ printf ' WHERE NOT EXISTS ('
+ printf ' SELECT *'
+ printf ' FROM `bl`'
+ printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
+ printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`'
+ printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`'
+ printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
+ printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ')'
+ printf ');\n'
+ } | \
+ ${mysql_command} --raw --batch | \
+ sed '1d' | \
+ sort -u
} | \
- sort | \
- uniq -d | \
- awk '{print $2}' | \
- sort -k1,1 | \
- join -1 1 -2 2 -o 2.1 - "${work_dir}/newest-dependencies" | \
- sort -u
- )
- black_listed_new=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${black_listed} ${black_listed} ${black_listed_new} | \
+ grep -vxF '' | \
sort | \
uniq -u
)
done
+
+black_listed=$(
+ printf '%s\n' "${black_listed}" | \
+ while read -r line; do
+ printf '%s' \
+ "${line}" | \
+ base64 -d
+ printf '\n'
+ done
+)
+
+deletion_list_count=$(
+ # shellcheck disable=SC2086
+ printf '%s\n' ${black_listed} | \
+ wc -l
+)
+if [ "${deletion_list_count}" -gt 1000 ]; then
+ >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
+ "${deletion_list_count}"
+ if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
+ {
+ printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
+ for repo_name in ${repo_names}; do
+ if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then
+ printf '%s: %s -> %s\n' \
+ "${repo_name}" \
+ "$(cat "${work_dir}/${repo_name}.revision")" \
+ "$(cat "${work_dir}/${repo_name}.revision.new")"
+ fi
+ done
+ } | \
+ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
+ sponge "${irc_dir}/#archlinux-ports/in"
+ fi
+ exit 3
+fi
+rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
+
echo "${black_listed}" | \
while read -r package; do
if [ -z "${package}" ]; then
@@ -599,32 +573,6 @@ fi
# update loop list in database
mysql_find_build_assignment_loops
-deletion_list_count=$(
- wc -l < \
- "${work_dir}/deletion-list.new"
-)
-if [ "${deletion_list_count}" -gt 1000 ]; then
- >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
- "${deletion_list_count}"
- if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
- {
- printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
- for repo_name in ${repo_names}; do
- if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then
- printf '%s: %s -> %s\n' \
- "${repo_name}" \
- "$(cat "${work_dir}/${repo_name}.revision")" \
- "$(cat "${work_dir}/${repo_name}.revision.new")"
- fi
- done
- } | \
- tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
- sponge "${irc_dir}/#archlinux-ports/in"
- fi
- exit 3
-fi
-rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
-
# Move the .new-files to the actual files
rm -rf --one-file-system "${work_dir}/build-list.loops"