summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2019-08-22 10:46:04 +0200
committerErich Eckner <git@eckner.net>2019-08-22 10:46:04 +0200
commit50ad9f8b90d941a9c58946bd4302036a457013b0 (patch)
tree7d598aa842f8676ab7166911f61a0b0027a8f50e
parent656f90ea25e6d1ac65f3d515cc8606519e36ae24 (diff)
downloadbuilder-50ad9f8b90d941a9c58946bd4302036a457013b0.tar.xz
bin/seed-build-list: general overhaul: blacklisting, un-deleting, ignore-list, architecture recognition, ...
-rwxr-xr-xbin/seed-build-list309
1 files changed, 113 insertions, 196 deletions
diff --git a/bin/seed-build-list b/bin/seed-build-list
index 61b187f..832641a 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -2,6 +2,17 @@
# shellcheck disable=SC2119,SC2120
+# seed the build list from various sources
+#
+# This uses the following schedule:
+# - find scheduleworthy packages by pkgname/pkgbase (diff to mirror,
+# broken dependencies, explicite regex) and their architecture
+# - find pkgbases of those packages (ask upstream, if necessary)
+# - remove all ignored packages (seed-igonre, explicitely ignored,
+# blacklist)
+# - run mysql_generate_package_metadata on each of them once (ignoring
+# the architecture for now)
+
# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"
@@ -9,10 +20,6 @@
# TODO: -a|--auto schedules too much - or is the scheduling really /that/ broken?
-# TODO: -u and -m should be merged
-
-# TODO: the blacklist is currently being ignored - why?
-
# shellcheck disable=SC2016
usage() {
>&2 echo ''
@@ -29,23 +36,20 @@ usage() {
>&2 echo ' Do not exit if mysql_generate_package_metadata() failed.'
>&2 echo ' -h|--help:'
>&2 echo ' Show this help and exit.'
- >&2 echo ' -i|--ignore $pkgbase:'
- >&2 echo ' Do not update the given package.'
+ >&2 echo ' -i|--ignore $pkgbase|$arch/$pkgbase:'
+ >&2 echo ' Do not update the given package (for the given $arch).'
>&2 echo ' -j|--jostle'
>&2 echo ' Give new build assignments the highest priority.'
>&2 echo ' -m|--mirror $url:'
- >&2 echo ' Schedule all packages, that are newer on the given'
- >&2 echo ' x86_64 mirror - except packages in'
- >&2 echo ' conf/seed-ignore-packages.'
+ >&2 echo ' Schedule all packages, that are newer (this includes'
+ >&2 echo ' packages currently unavailable on archlinux32) on the'
+ >&2 echo ' given x86_64 mirror - except packages on the blacklist'
+ >&2 echo ' or in conf/seed-ignore-packages.'
>&2 echo ' -n|--no-action:'
>&2 echo ' Do not actually update build-list, just print it.'
>&2 echo ' -p|--package $pkg_regex:'
>&2 echo ' Reschedule packages with matching pkgname or pkgbase.'
>&2 echo ' Note, that these packages must be known to the database.'
- >&2 echo ' -u|--undelete $url: '
- >&2 echo ' Schedule all former deletion-list packages which do not'
- >&2 echo ' belong on the deletion-list anymore (e.g. can be built)'
- >&2 echo ' and which are available on the given x86_64 mirror.'
>&2 echo ' -w|--wait:'
>&2 echo ' Wait for lock if necessary.'
[ -z "$1" ] && exit 1 || exit "$1"
@@ -56,7 +60,7 @@ tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT
eval set -- "$(
- getopt -o afhi:jm:np:u:w \
+ getopt -o afhi:jm:np:w \
--long auto \
--long force \
--long help \
@@ -65,7 +69,6 @@ eval set -- "$(
--long mirror: \
--long no-action \
--long package: \
- --long undelete: \
--long wait \
-n "$(basename "$0")" -- "$@" || \
echo usage
@@ -77,11 +80,6 @@ jostle=false
update=true
wait_for_lock='-n'
-if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
- sed 's/^/2\t/' "${base_dir}/conf/seed-ignore-packages" >> \
- "${tmp_dir}/ignore-packages"
-fi
-
while true
do
case "$1" in
@@ -96,8 +94,13 @@ do
;;
-i|--ignore)
shift
- printf '3\t%s\n' "$1" >> \
- "${tmp_dir}/ignore-packages"
+ if [ -z "${1%%*/*}" ]; then
+ printf '%s\n' "$1" \
+ | tr '/' '\t'
+ else
+ printf 'any\t%s\n' "$1"
+ fi \
+ >> "${tmp_dir}/ignore-packages"
;;
-j|--jostle)
jostle=true
@@ -115,11 +118,6 @@ do
printf '%s\n' "$1" >> \
"${tmp_dir}/package-regexes"
;;
- -u|--undelete)
- shift
- printf '%s\n' "$1" >> \
- "${tmp_dir}/undelete-mirrors"
- ;;
-w|--wait)
wait_for_lock=''
;;
@@ -142,7 +140,6 @@ fi
if [ ! -s "${tmp_dir}/mirrors" ] && \
[ ! -s "${tmp_dir}/package-regexes" ] && \
- [ ! -s "${tmp_dir}/undelete-mirrors" ] && \
! ${auto}; then
# nothing to do
>&2 echo 'No options given to do anything.'
@@ -178,7 +175,30 @@ repos=$(
printf '%s\n' 'multilib'
)
-# TODO: translate lib32-* packages instead of ignoring them for -u and -m
+if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
+ sed '
+ s@/@\t@
+ t
+ s/^/any\t/
+ ' "${base_dir}/conf/seed-ignore-packages" >> \
+ "${tmp_dir}/ignore-packages"
+fi
+
+# shellcheck disable=SC2154
+git -C "${repo_paths__archlinux32}" archive "${repo_heads__archlinux32}" -- blacklist \
+| tar -t \
+| sed '
+ s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
+ t
+ d
+' \
+>> "${tmp_dir}/ignore-packages"
+
+expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" \
+< "${tmp_dir}/ignore-packages" \
+| sponge "${tmp_dir}/ignore-packages"
+
+# TODO: translate lib32-* packages instead of ignoring them for -m
# harvest pkgnames from mirror delta
if [ -s "${tmp_dir}/mirrors" ]; then
@@ -196,7 +216,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then
N
s/^.*\n//
/^lib32-/d
- s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/
+ s/^\(.*-\)x86_64\.pkg\.tar\.xz$/\1i486\n\1i686\n\1pentium4/
+ s/^\(.*-any\)\.pkg\.tar\.xz$/\1/
' | \
sed '
s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/
@@ -212,12 +233,14 @@ if [ -s "${tmp_dir}/mirrors" ]; then
printf ' FROM `binary_packages`'
mysql_join_binary_packages_architectures
mysql_join_binary_packages_binary_packages_in_repositories
- printf ' WHERE NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
+ mysql_join_binary_packages_in_repositories_repositories
+ printf ' WHERE `repositories`.`is_on_master_mirror`'
+ printf ' OR `repositories`.`name`="build-list"'
} | \
mysql_run_query | \
sed '
- s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)$/ours \2 \4 \1/
- s/^\(.* \)any\(\.\S\+ \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/
+ s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)\.pkg\.tar\.xz$/ours \2 \4 \1/
+ s/^\(.* \)any\( \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/
'
} | \
expand_version 2 | \
@@ -227,48 +250,13 @@ if [ -s "${tmp_dir}/mirrors" ]; then
sed -n '
s/^theirs \(\S\+ \)//
T
- s/^\(\S\+ \)//
- T
p
' | \
- if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
- grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
+ if [ -r "${tmp_dir}/ignore-packages" ]; then
+ grep -vxFf "${tmp_dir}/ignore-packages"
else
cat
fi | \
- sed 's/^/1\t/' | \
- sort -u >> \
- "${tmp_dir}/pkgnames"
-fi
-
-# harvest pkgnames from undelete-mirrors
-if [ -s "${tmp_dir}/undelete-mirrors" ]; then
- while read -r mirror; do
- if [ -z "${mirror}" ]; then
- continue
- fi
- for repo in ${repos}; do
- curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
- tar -Oxz --wildcards '*/desc' | \
- sed '
- /^%FILENAME%$/!d
- N
- s/^.*\n//
- /^lib32-/d
- s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/
- ' | \
- sed '
- s/^\(.*\)\(-[^-]\+\)\{3\}$/\1/
- '
- done
- done < \
- "${tmp_dir}/undelete-mirrors" | \
- if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
- grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
- else
- cat
- fi | \
- sed 's/^/2\t/' | \
sort -u >> \
"${tmp_dir}/pkgnames"
fi
@@ -294,14 +282,15 @@ if [ -s "${tmp_dir}/package-regexes" ]; then
printf ' ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n'
} | \
mysql_run_query | \
+ tr '\t' ' ' | \
sed -n '
- /^pkgbase\s/ {
- s/^\S\+\s/3\t/
+ /^pkgbase / {
+ s/^\S\+ /any /
w /dev/stdout
d
}
- /^pkgname\s/ {
- s/^\S\+\s/3\t/
+ /^pkgname / {
+ s/^\S\+ /any /
w /dev/stderr
d
}
@@ -315,14 +304,18 @@ fi
# create pkgbases to given pkgnames
if [ -s "${tmp_dir}/pkgnames" ]; then
printf 'CREATE TEMPORARY TABLE `pkgnames` ('
- printf '`priority` SMALLINT,'
+ printf '`architecture` VARCHAR(16),'
printf '`pkgname` VARCHAR(64),'
- printf 'UNIQUE KEY `pkgname`(`pkgname`)'
+ printf 'UNIQUE KEY `content`(`architecture`,`pkgname`)'
printf ');\n'
- printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` (`priority`,`pkgname`);\n' \
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` COLUMNS TERMINATED BY " " (`architecture`,`pkgname`);\n' \
"${tmp_dir}/pkgnames"
- printf 'SELECT DISTINCT "pkgbase",`pkgnames`.`priority`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
+ printf 'SELECT DISTINCT'
+ printf ' "pkgbase",'
+ printf '`pkgnames`.`architecture`,'
+ printf '`package_sources`.`pkgbase`,'
+ printf '`upstream_repositories`.`name`'
printf ' FROM `pkgnames`'
printf ' JOIN `binary_packages`'
printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`'
@@ -331,7 +324,10 @@ fi
mysql_join_package_sources_upstream_repositories
printf ';\n'
- printf 'SELECT DISTINCT "pkgname",`pkgnames`.`priority`,`pkgnames`.`pkgname`'
+ printf 'SELECT DISTINCT'
+ printf ' "pkgname",'
+ printf '`pkgnames`.`architecture`,'
+ printf '`pkgnames`.`pkgname`'
printf ' FROM `package_sources`'
mysql_join_package_sources_upstream_repositories
mysql_join_package_sources_build_assignments
@@ -344,6 +340,8 @@ fi
fi
# auto-detect pkgbases
if ${auto}; then
+ # TODO: check, if this still works or if it relied on the `priority` magic
+ >&2 echo 'autoscheduling needs testing / inspection - be prepared to schedule too many packages this way!'
# schedule any package, that:
# 1) is not on the build-list currently and
# 2) has some dependency which is not provided by any package which
@@ -413,7 +411,7 @@ fi
# 1)
printf ' WHERE `least_stable_bp`.`is_on_build_list`;'
- printf 'SELECT DISTINCT "pkgbase",1,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
+ printf 'SELECT DISTINCT "pkgbase",`ba_a`.`name`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
printf ' FROM `least_stable_bp`'
printf ' JOIN `binary_packages`'
printf ' ON `least_stable_bp`.`id`=`binary_packages`.`id`'
@@ -422,6 +420,7 @@ fi
printf ' AND `dependency_types`.`relevant_for_binary_packages`'
mysql_join_dependencies_versions '' 'd_v'
mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_architectures '' 'ba_a'
mysql_join_build_assignments_package_sources
mysql_join_package_sources_upstream_repositories
# some dependencies are not provided by the least stable packages
@@ -456,14 +455,15 @@ fi
fi
} | \
mysql_run_query | \
+ tr '\t' ' ' | \
sed -n '
- /^pkgbase\s/ {
- s/^\S\+\s//
+ /^pkgbase / {
+ s/^\S\+ //
w /dev/stderr
d
}
- /^pkgname\s/ {
- s/^\S\+\s//
+ /^pkgname / {
+ s/^\S\+ //
w /dev/stdout
d
}
@@ -486,7 +486,7 @@ done
# pkgnames -> pkgbases (with help from upstream)
if [ -s "${tmp_dir}/pkgnames" ]; then
- while read -r priority pkgname; do
+ while read -r architecture pkgname; do
content=$(
curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
tr ',' '\n'
@@ -512,11 +512,11 @@ if [ -s "${tmp_dir}/pkgnames" ]; then
'
)
if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
- printf '%s\t%s\n' "${priority}" "${pkgname}"
+ printf '%s %s\n' "${architecture}" "${pkgname}"
continue
fi
- printf '%s\t%s\t%s\n' \
- "${priority}" \
+ printf '%s %s %s\n' \
+ "${architecture}" \
"${pkgbase}" \
"${repo}" >> \
"${tmp_dir}/pkgbases"
@@ -531,112 +531,30 @@ if [ -s "${tmp_dir}/pkgnames" ]; then
exit 2
fi
-# now we (re)schedule all pkgbases which are:
-# - not explicitely ignored (priority 3) <- explicitely scheduled
-# - not explicitely/implicitely ignored, but not currently available
-# (priority 2) <- from undelete-mirror-delta
-# - neither ignored nor to-be-deleted (priority 1) <- from mirror-delta or auto-rebuild
+sort -u "${tmp_dir}/pkgbases" \
+| sort -k3,3 \
+| sponge "${tmp_dir}/pkgbases"
+
+# get the current HEADs
# shellcheck disable=SC2016
{
- printf 'CREATE TEMPORARY TABLE `ignore_packages` ('
- printf '`priority` SMALLINT,'
- printf '`pkgbase` VARCHAR(64),'
- printf 'UNIQUE KEY `content`(`priority`,`pkgbase`)'
- printf ');\n'
- printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `ignore_packages`(`priority`,`pkgbase`);\n' \
- "${tmp_dir}/ignore-packages"
-
- printf 'CREATE TEMPORARY TABLE `pkgbases` ('
- printf '`priority` SMALLINT,'
- printf '`pkgbase` VARCHAR(64),'
- printf '`repo` VARCHAR(64),'
- printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
- printf ');\n'
- printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`priority`,`pkgbase`,`repo`);\n' \
- "${tmp_dir}/pkgbases"
-
- printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
- printf ' SELECT 2,`sub_q`.`pkgbase`'
- printf ' FROM ('
- printf 'SELECT'
- printf ' `package_sources`.`pkgbase`,'
- printf 'GROUP_CONCAT('
- printf 'DISTINCT'
- printf ' `build_assignments`.`architecture`'
- printf ' ORDER BY `build_assignments`.`architecture`'
- printf ') AS `architectures`'
- printf ' FROM `package_sources`'
- mysql_join_package_sources_build_assignments
- mysql_join_build_assignments_binary_packages
- mysql_join_binary_packages_binary_packages_in_repositories
- mysql_join_binary_packages_in_repositories_repositories
- printf ' WHERE `build_assignments`.`is_black_listed` IS NOT NULL'
- printf ' OR `package_sources`.`pkgbase` LIKE "lib32-%%"'
- printf ' OR `repositories`.`is_on_master_mirror`'
- printf ' OR `binary_packages_in_repositories`.`repository`=%s' \
- "${repository_ids__any_build_list}"
- printf ' GROUP BY `package_sources`.`pkgbase`'
- printf ') AS `sub_q`'
- # shellcheck disable=SC2154
- printf ' WHERE `sub_q`.`architectures` IN ("%s","%s","%s,%s")' \
- "${architecture_ids__any}" \
- "${non_any_architecture_ids}" \
- "${architecture_ids__any}" \
- "${non_any_architecture_ids}"
- printf ';\n'
-
- printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
- printf ' SELECT 1,`package_sources`.`pkgbase`'
- printf ' FROM `package_sources`'
- mysql_join_package_sources_build_assignments
- mysql_join_build_assignments_binary_packages
- mysql_join_binary_packages_binary_packages_in_repositories
- printf ' WHERE `binary_packages_in_repositories`.`is_to_be_deleted`'
- printf ';\n'
-
- printf 'SELECT '
- printf '`pkgbases`.`pkgbase`,'
- printf '`git_repositories`.`head`,'
- printf '('
- printf 'SELECT `al32`.`head`'
- printf ' FROM `git_repositories` AS `al32`'
- printf ' WHERE `al32`.`name`="archlinux32"'
- printf '),'
- printf '`pkgbases`.`repo`'
- printf ' FROM `pkgbases`'
- printf ' JOIN `upstream_repositories`'
- printf ' ON `upstream_repositories`.`name`=`pkgbases`.`repo`'
+ printf 'SELECT'
+ printf ' `upstream_repositories`.`name`,'
+ printf '`git_repositories`.`head`'
+ printf ' FROM `upstream_repositories`'
mysql_join_upstream_repositories_git_repositories
- printf ' WHERE NOT EXISTS ('
- printf 'SELECT 1'
- printf ' FROM `ignore_packages`'
- printf ' WHERE `ignore_packages`.`pkgbase`=`pkgbases`.`pkgbase`'
- printf ' AND `ignore_packages`.`priority`>=`pkgbases`.`priority`'
- printf ') AND ('
- printf '`pkgbases`.`priority`!=2'
- printf ' OR NOT EXISTS ('
- printf 'SELECT 1'
- printf ' FROM `package_sources`'
- mysql_join_package_sources_build_assignments
- mysql_join_build_assignments_binary_packages
- mysql_join_binary_packages_binary_packages_in_repositories
- printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
- mysql_join_binary_packages_in_repositories_repositories
- printf ' AND ('
- printf '`repositories`.`is_on_master_mirror`'
- printf ' OR `repositories`.`name`="build-list"'
- printf ') WHERE `package_sources`.`pkgbase`=`pkgbases`.`pkgbase`'
- printf ')'
- printf ');\n'
-} | \
- mysql_run_query | \
- tr '\t' ' ' | \
- sort -u > \
- "${tmp_dir}/update-list"
+} \
+| mysql_run_query \
+| tr '\t' ' ' \
+| sort -k1,1 \
+| join -1 3 -2 1 -o 1.1,1.2,1.3,2.2 "${tmp_dir}/pkgbases" - \
+| sponge "${tmp_dir}/pkgbases"
if ${update}; then
- while read -r pkgbase git_rev mod_git_rev repo; do
+ cut -d' ' -f 2,3,4 "${tmp_dir}/pkgbases" \
+ | sort -u \
+ | while read -r pkgbase repo git_rev; do
success=false
# shellcheck disable=SC2154
for gr_r in \
@@ -644,9 +562,9 @@ if ${update}; then
"${repo_heads__packages}:core" \
"${repo_heads__packages}:extra" \
"${repo_heads__community}:community"; do
- printf '%s ' "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}" | \
+ printf '%s ' "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}" | \
sed 's/ $/\n/'
- if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}"; then
+ if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}"; then
success=true
break
fi
@@ -655,13 +573,12 @@ if ${update}; then
! ${ignore_mysql_generate_package_metadata_errors}; then
exit 2
fi
- done < \
- "${tmp_dir}/update-list"
+ done
cut -d' ' -f1 < \
- "${tmp_dir}/update-list" | \
+ "${tmp_dir}/pkgbases" | \
sort -u | \
- sponge "${tmp_dir}/update-list"
+ sponge "${tmp_dir}/pkgbases"
# shellcheck disable=SC2016
{
@@ -670,7 +587,7 @@ if ${update}; then
printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
printf ');\n'
printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`pkgbase`);\n' \
- "${tmp_dir}/update-list"
+ "${tmp_dir}/pkgbases"
printf 'DELETE `d_bpir`'
printf ' FROM `pkgbases`'
@@ -723,5 +640,5 @@ if ${update}; then
mysql_query_remove_old_binary_packages_from_build_list | \
mysql_run_query
else
- cat "${tmp_dir}/update-list"
+ cat "${tmp_dir}/pkgbases"
fi