summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2017-09-09 20:59:57 +0200
committerErich Eckner <git@eckner.net>2017-09-09 20:59:57 +0200
commit9c54adc41ea3840a39202ccb95d0ef5b79908521 (patch)
tree056296c202fef34537902353a1540e8a48bc01a8
parentac46c928bcd779ec0b230061215ad66434ff7765 (diff)
downloadbuilder-9c54adc41ea3840a39202ccb95d0ef5b79908521.tar.xz
bin/seed-build-list: bugfixes and cleanups
-rwxr-xr-xbin/seed-build-list197
1 files changed, 86 insertions, 111 deletions
diff --git a/bin/seed-build-list b/bin/seed-build-list
index 998ca21..4067ed0 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -43,10 +43,9 @@ eval set -- "$(
)"
touch "${tmp_dir}/mirrors"
-touch "${tmp_dir}/delta_packages"
-touch "${tmp_dir}/delta_pkgbases"
-touch "${tmp_dir}/package_regexes"
-touch "${tmp_dir}/ignore_packages"
+touch "${tmp_dir}/delta-packages"
+touch "${tmp_dir}/package-regexes"
+touch "${tmp_dir}/ignore-packages"
update=true
@@ -59,7 +58,7 @@ do
-i|--ignore)
shift
echo "$1" >> \
- "${tmp_dir}/ignore_packages"
+ "${tmp_dir}/ignore-packages"
;;
-m|--mirror)
shift
@@ -72,7 +71,7 @@ do
-p|--package)
shift
echo "$1" >> \
- "${tmp_dir}/package_regexes"
+ "${tmp_dir}/package-regexes"
;;
--)
shift
@@ -90,36 +89,61 @@ if [ $# -ne 0 ]; then
usage 1
fi
-repos='core community extra gnome-unstable kde-unstable'
+repos="${stable_package_repositories}"
-for repo in ${repo_names}; do
- if [ "${repo_names}" = 'archlinux32' ]; then
- continue
- fi
- eval 'repo_path="${repo_paths__'"${repo}"'}"'
- git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
- tar -t | \
- grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
- grep -v -- '-i686/PKGBUILD$' | \
- grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
- sed '
- s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2|
- '
-done | \
+find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f\n' | \
+ sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
sort -k1,1 > \
- "${tmp_dir}/known_packages"
+ "${tmp_dir}/known-packages"
-# generate delta_packages from package_regexes
+mod_repo_rev=$(cat "${work_dir}/archlinux32.revision")
+{
+ for repo in ${repo_names}; do
+ if [ "${repo_names}" = 'archlinux32' ]; then
+ continue
+ fi
+ eval 'repo_path="${repo_paths__'"${repo}"'}"'
+ repo_rev=$(cat "${work_dir}/${repo}.revision")
+ git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
+ tar -t | \
+ grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
+ grep -v -- '-i686/PKGBUILD$' | \
+ grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
+ sed '
+ s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|'"${repo_rev}"' \1 \2|
+ '
+ done | \
+ sort -u | \
+ sort -k1,1
+ awk '{print "nothing " $1 " " $4}' "${tmp_dir}/known-packages" | \
+ sort -u | \
+ sed 'p'
+} | \
+ sort -k2,3 | \
+ uniq -uf1 | \
+ while read -r repo_rev pkg prepo; do
+ generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}"
+ done
+
+find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
+ sed '
+ s|^.*/||
+ s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 |
+ ' | \
+ sort -k5,5 > \
+ "${tmp_dir}/known-packages"
+
+# generate delta-packages from package-regexes
while read -r pkg_regex; do
if [ -z "${pkg_regex}" ]; then
continue
fi
- awk '{print $1}' "${tmp_dir}/known_packages" | \
+ awk '{print $5}' "${tmp_dir}/known-packages" | \
grep "${pkg_regex}" || \
true
done < \
- "${tmp_dir}/package_regexes" >> \
- "${tmp_dir}/delta_packages"
+ "${tmp_dir}/package-regexes" >> \
+ "${tmp_dir}/delta-packages"
# genereate delta_packages from mirror delta
@@ -141,101 +165,50 @@ while read -r mirror; do
# ours
for repo in $(ls_master_mirror 'i686'); do
- ls_master_mirror "i686/$repo" | \
+ ls_master_mirror "i686/${repo}" | \
grep '\.pkg\.tar\.xz$' | \
sed 's|-[^-]\+$||'
done | \
sed '
s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1|
'
- ) | \
+ ) | \
sort -k4,4 -k3Vr,3 -k1,1 | \
uniq -f3 | \
grep '^theirs ' | \
awk '{print $4}' | \
sort -k1,1 >> \
- "${tmp_dir}/delta_packages"
+ "${tmp_dir}/delta-packages"
done < \
"${tmp_dir}/mirrors"
-sort -k1,1 -u "${tmp_dir}/delta_packages" | \
- sponge "${tmp_dir}/delta_packages"
+sort -k1,1 -u "${tmp_dir}/delta-packages" | \
+ sponge "${tmp_dir}/delta-packages"
-join -j1 "${tmp_dir}/delta_packages" "${tmp_dir}/known_packages" >> \
- "${tmp_dir}/append_packages"
+join -1 1 -2 5 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/delta-packages" "${tmp_dir}/known-packages" >> \
+ "${tmp_dir}/append-packages"
(
- awk '{print $1}' "${tmp_dir}/append_packages"
- cat "${tmp_dir}/delta_packages"
+ awk '{print $5}' "${tmp_dir}/append-packages"
+ cat "${tmp_dir}/delta-packages"
) | \
sort | \
uniq -u | \
- sponge "${tmp_dir}/delta_packages"
+ sponge "${tmp_dir}/delta-packages"
-while read -r pkg; do
- if [ -z "${pkg}" ]; then
- continue
- fi
- pkgbase=$(
- curl -sS "https://www.archlinux.org/packages/community/x86_64/${pkg}/" | \
- sed -n '
- /Base Package:/{
- :a
- N
- s|</tr>.*$||
- Ta
- p
- }
- ' | \
- tr '<>' '\n' | \
- grep '^a href="' | \
- sed '
- s|^a href="||
- s|".*||
- s|/$||
- s|^.*/||
- '
- )
- if [ -n "${pkgbase}" ]; then
- echo "${pkgbase}" >> \
- "${tmp_dir}/delta_pkgbases"
- else
- echo "${pkg}"
- fi
-done < \
- "${tmp_dir}/delta_packages" | \
- sponge "${tmp_dir}/delta_packages"
-
-sort -u "${tmp_dir}/delta_pkgbases" | \
- sponge "${tmp_dir}/delta_pkgbases"
-
-join -j1 "${tmp_dir}/delta_pkgbases" "${tmp_dir}/known_packages" >> \
- "${tmp_dir}/append_packages"
-
-sort -u "${tmp_dir}/append_packages" | \
- sponge "${tmp_dir}/append_packages"
-
-rm "${tmp_dir}/delta_pkgbases"
-
-if [ -s "${tmp_dir}/delta_packages" ]; then
+if [ -s "${tmp_dir}/delta-packages" ]; then
echo 'There are still packages newer for x86_64 which I cannot identify :-/'
+ cat "${tmp_dir}/delta-packages"
exit 2
fi
-rm "${tmp_dir}/delta_packages"
+rm "${tmp_dir}/delta-packages"
-# convert append_packages to build-list.new (add git hashes)
+# convert append-packages to build-list.new (add git hashes)
-while read -r pkg repo; do
- git_repo=$(find_git_repository_to_package_repository "${repo}")
- printf '%s %s %s %s\n' \
- "${pkg}" \
- "$(cat "${work_dir}/${git_repo}.revision")" \
- "$(cat "${work_dir}/archlinux32.revision")" \
- "${repo}"
-done < \
- "${tmp_dir}/append_packages" | \
- awk '{print "append " $1 " " $2 " " $3 " " $4 " " $1}' > \
+cut -d' ' -f1,2,3,4 < \
+ "${tmp_dir}/append-packages" | \
+ sort -u > \
"${tmp_dir}/build-list.append"
# Create a lock file for build list.
@@ -246,38 +219,40 @@ if ${update}; then
fi
{
- # ignore packages already on the build list
+ awk '{print $2 " " $3 " " $4 " " $1}' "${tmp_dir}/build-list.append"
+
+ # ignore packages on the build-list
grep -vxF 'break_loops' "${work_dir}/build-list" | \
- awk '{print "old " $1 " " $2 " " $3 " " $4 " " $1}'
+ awk '{print $2 " " $3 " " $4 " " $1}' | \
+ sed 'p'
# ignore packages on the deletion-list
- awk '{print "delete " $1 " X X X " $1}' "${work_dir}/deletion-list"
+ awk '{print "X X X " $1}' "${work_dir}/deletion-list" | \
+ sed 'p'
# ignore packages on the blacklist
git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
tar -Ox blacklist | \
- awk '{print "blacklist " $1 " X X X " $1}'
+ awk '{print "X X X " $1}' | \
+ sed 'p'
# ignore explicitely ignored packages
- awk '{print "ignore " $1 " X X X " $1}' "${tmp_dir}/ignore_packages"
-} > \
- "${tmp_dir}/build-list.old"
+ awk '{print "X X X " $1}' "${tmp_dir}/ignore-packages" | \
+ sed 'p'
+} | \
+ sort -k4,4 | \
+ uniq -uf3 > \
+ "${tmp_dir}/build-list.new"
if ${update}; then
- cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \
- sort -k6,6 -k1,1 | \
- uniq -uf5 | \
- grep '^append ' | \
- awk '{print $2 " " $3 " " $4 " " $5}' >> \
+ awk '{print $4 " " $1 " " $2 " " $3}' \
+ "${tmp_dir}/build-list.new" >> \
"${work_dir}/build-list"
# Remove the lock file
rm -f "${build_list_lock_file}"
else
- cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \
- sort -k6,6 | \
- uniq -uf5 | \
- grep '^append ' | \
- awk '{print $2 " " $3 " " $4 " " $5}'
+ awk '{print $4 " " $1 " " $2 " " $3}' \
+ "${tmp_dir}/build-list.new"
fi