From 9c54adc41ea3840a39202ccb95d0ef5b79908521 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Sat, 9 Sep 2017 20:59:57 +0200
Subject: bin/seed-build-list: bugfixes and cleanups

---
 bin/seed-build-list | 197 +++++++++++++++++++++++-----------------------------
 1 file changed, 86 insertions(+), 111 deletions(-)

(limited to 'bin/seed-build-list')

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 998ca21..4067ed0 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -43,10 +43,9 @@ eval set -- "$(
   )"
 
 touch "${tmp_dir}/mirrors"
-touch "${tmp_dir}/delta_packages"
-touch "${tmp_dir}/delta_pkgbases"
-touch "${tmp_dir}/package_regexes"
-touch "${tmp_dir}/ignore_packages"
+touch "${tmp_dir}/delta-packages"
+touch "${tmp_dir}/package-regexes"
+touch "${tmp_dir}/ignore-packages"
 
 update=true
 
@@ -59,7 +58,7 @@ do
     -i|--ignore)
       shift
       echo "$1" >> \
-        "${tmp_dir}/ignore_packages"
+        "${tmp_dir}/ignore-packages"
     ;;
     -m|--mirror)
       shift
@@ -72,7 +71,7 @@ do
     -p|--package)
       shift
       echo "$1" >> \
-        "${tmp_dir}/package_regexes"
+        "${tmp_dir}/package-regexes"
     ;;
     --)
       shift
@@ -90,36 +89,61 @@ if [ $# -ne 0 ]; then
   usage 1
 fi
 
-repos='core community extra gnome-unstable kde-unstable'
+repos="${stable_package_repositories}"
 
-for repo in ${repo_names}; do
-  if [ "${repo_names}" = 'archlinux32' ]; then
-    continue
-  fi
-  eval 'repo_path="${repo_paths__'"${repo}"'}"'
-  git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
-    tar -t | \
-    grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
-    grep -v -- '-i686/PKGBUILD$' | \
-    grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
-    sed '
-      s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2|
-    '
-done | \
+find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f\n' | \
+  sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
   sort -k1,1 > \
-  "${tmp_dir}/known_packages"
+  "${tmp_dir}/known-packages"
 
-# generate delta_packages from package_regexes
+mod_repo_rev=$(cat "${work_dir}/archlinux32.revision")
+{
+  for repo in ${repo_names}; do
+    if [ "${repo_names}" = 'archlinux32' ]; then
+      continue
+    fi
+    eval 'repo_path="${repo_paths__'"${repo}"'}"'
+    repo_rev=$(cat "${work_dir}/${repo}.revision")
+    git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
+      tar -t | \
+      grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
+      grep -v -- '-i686/PKGBUILD$' | \
+      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
+      sed '
+        s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|'"${repo_rev}"' \1 \2|
+      '
+  done | \
+    sort -u | \
+    sort -k1,1
+  awk '{print "nothing " $1 " " $4}' "${tmp_dir}/known-packages" | \
+    sort -u | \
+    sed 'p'
+} | \
+  sort -k2,3 | \
+  uniq -uf1 | \
+  while read -r repo_rev pkg prepo; do
+    generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}"
+  done
+
+find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
+  sed '
+    s|^.*/||
+    s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 |
+  ' | \
+  sort -k5,5 > \
+  "${tmp_dir}/known-packages"
+
+# generate delta-packages from package-regexes
 while read -r pkg_regex; do
   if [ -z "${pkg_regex}" ]; then
     continue
   fi
-  awk '{print $1}' "${tmp_dir}/known_packages" | \
+  awk '{print $5}' "${tmp_dir}/known-packages" | \
     grep "${pkg_regex}" || \
     true
 done < \
-  "${tmp_dir}/package_regexes" >> \
-  "${tmp_dir}/delta_packages"
+  "${tmp_dir}/package-regexes" >> \
+  "${tmp_dir}/delta-packages"
 
 # genereate delta_packages from mirror delta
 
@@ -141,101 +165,50 @@ while read -r mirror; do
 
     # ours
     for repo in $(ls_master_mirror 'i686'); do
-      ls_master_mirror "i686/$repo" | \
+      ls_master_mirror "i686/${repo}" | \
         grep '\.pkg\.tar\.xz$' | \
         sed 's|-[^-]\+$||'
     done | \
       sed '
         s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1|
       '
-    ) | \
+  ) | \
     sort -k4,4 -k3Vr,3 -k1,1 | \
     uniq -f3 | \
     grep '^theirs ' | \
     awk '{print $4}' | \
     sort -k1,1 >> \
-    "${tmp_dir}/delta_packages"
+    "${tmp_dir}/delta-packages"
 done < \
   "${tmp_dir}/mirrors"
 
-sort -k1,1 -u "${tmp_dir}/delta_packages" | \
-  sponge "${tmp_dir}/delta_packages"
+sort -k1,1 -u "${tmp_dir}/delta-packages" | \
+  sponge "${tmp_dir}/delta-packages"
 
-join -j1 "${tmp_dir}/delta_packages" "${tmp_dir}/known_packages" >> \
-  "${tmp_dir}/append_packages"
+join -1 1 -2 5 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/delta-packages" "${tmp_dir}/known-packages" >> \
+  "${tmp_dir}/append-packages"
 
 (
-  awk '{print $1}' "${tmp_dir}/append_packages"
-  cat "${tmp_dir}/delta_packages"
+  awk '{print $5}' "${tmp_dir}/append-packages"
+  cat "${tmp_dir}/delta-packages"
 ) | \
   sort | \
   uniq -u | \
-  sponge "${tmp_dir}/delta_packages"
+  sponge "${tmp_dir}/delta-packages"
 
-while read -r pkg; do
-  if [ -z "${pkg}" ]; then
-    continue
-  fi
-  pkgbase=$(
-    curl -sS "https://www.archlinux.org/packages/community/x86_64/${pkg}/" | \
-      sed -n '
-        /Base Package:/{
-          :a
-          N
-          s|</tr>.*$||
-          Ta
-          p
-        }
-      ' | \
-      tr '<>' '\n' | \
-      grep '^a href="' | \
-      sed '
-        s|^a href="||
-        s|".*||
-        s|/$||
-        s|^.*/||
-      '
-  )
-  if [ -n "${pkgbase}" ]; then
-    echo "${pkgbase}" >> \
-      "${tmp_dir}/delta_pkgbases"
-  else
-    echo "${pkg}"
-  fi
-done < \
-  "${tmp_dir}/delta_packages" | \
-  sponge "${tmp_dir}/delta_packages"
-
-sort -u "${tmp_dir}/delta_pkgbases" | \
-  sponge "${tmp_dir}/delta_pkgbases"
-
-join -j1 "${tmp_dir}/delta_pkgbases" "${tmp_dir}/known_packages" >> \
-  "${tmp_dir}/append_packages"
-
-sort -u "${tmp_dir}/append_packages" | \
-  sponge "${tmp_dir}/append_packages"
-
-rm "${tmp_dir}/delta_pkgbases"
-
-if [ -s "${tmp_dir}/delta_packages" ]; then
+if [ -s "${tmp_dir}/delta-packages" ]; then
   echo 'There are still packages newer for x86_64 which I cannot identify :-/'
+  cat "${tmp_dir}/delta-packages"
   exit 2
 fi
 
-rm "${tmp_dir}/delta_packages"
+rm "${tmp_dir}/delta-packages"
 
-# convert append_packages to build-list.new (add git hashes)
+# convert append-packages to build-list.new (add git hashes)
 
-while read -r pkg repo; do
-  git_repo=$(find_git_repository_to_package_repository "${repo}")
-  printf '%s %s %s %s\n' \
-    "${pkg}" \
-    "$(cat "${work_dir}/${git_repo}.revision")" \
-    "$(cat "${work_dir}/archlinux32.revision")" \
-    "${repo}"
-done < \
-  "${tmp_dir}/append_packages" | \
-  awk '{print "append " $1 " " $2 " " $3 " " $4 " " $1}' > \
+cut -d' ' -f1,2,3,4 < \
+  "${tmp_dir}/append-packages" | \
+  sort -u > \
   "${tmp_dir}/build-list.append"
 
 # Create a lock file for build list.
@@ -246,38 +219,40 @@ if ${update}; then
 fi
 
 {
-  # ignore packages already on the build list
+  awk '{print $2 " " $3 " " $4 " " $1}' "${tmp_dir}/build-list.append"
+
+  # ignore packages on the build-list
   grep -vxF 'break_loops' "${work_dir}/build-list" | \
-    awk '{print "old " $1 " " $2 " " $3 " " $4 " " $1}'
+    awk '{print $2 " " $3 " " $4 " " $1}' | \
+    sed 'p'
 
   # ignore packages on the deletion-list
-  awk '{print "delete " $1 " X X X " $1}' "${work_dir}/deletion-list"
+  awk '{print "X X X " $1}' "${work_dir}/deletion-list" | \
+    sed 'p'
 
   # ignore packages on the blacklist
   git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
     tar -Ox blacklist | \
-    awk '{print "blacklist " $1 " X X X " $1}'
+    awk '{print "X X X " $1}' | \
+    sed 'p'
 
   # ignore explicitely ignored packages
-  awk '{print "ignore " $1 " X X X " $1}' "${tmp_dir}/ignore_packages"
-} > \
-  "${tmp_dir}/build-list.old"
+  awk '{print "X X X " $1}' "${tmp_dir}/ignore-packages" | \
+    sed 'p'
+} | \
+  sort -k4,4 | \
+  uniq -uf3 > \
+  "${tmp_dir}/build-list.new"
 
 if ${update}; then
-  cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \
-    sort -k6,6 -k1,1 | \
-    uniq -uf5 | \
-    grep '^append ' | \
-    awk '{print $2 " " $3 " " $4 " " $5}' >> \
+  awk '{print $4 " " $1 " " $2 " " $3}' \
+    "${tmp_dir}/build-list.new" >> \
     "${work_dir}/build-list"
 
   # Remove the lock file
 
   rm -f "${build_list_lock_file}"
 else
-  cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \
-    sort -k6,6 | \
-    uniq -uf5 | \
-    grep '^append ' | \
-    awk '{print $2 " " $3 " " $4 " " $5}'
+  awk '{print $4 " " $1 " " $2 " " $3}' \
+    "${tmp_dir}/build-list.new"
 fi
-- 
cgit v1.2.3-54-g00ecf