summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2017-04-22 23:13:20 +0200
committerErich Eckner <git@eckner.net>2017-04-23 00:05:12 +0200
commit563103068d7b669227d3794eec3b5e32b7b124d6 (patch)
tree26f8775f3ec2ef2f4b2f926e50a127ac8e93709b
parent8d42512ab208f45a62688dcade6c39f6ca16c430 (diff)
downloadbuilder-563103068d7b669227d3794eec3b5e32b7b124d6.tar.xz
use mksrcinfo and tsort
-rwxr-xr-xbin/get-package-updates302
-rw-r--r--conf/.gitignore1
-rwxr-xr-xconf/default.conf41
3 files changed, 159 insertions, 185 deletions
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 572053f..50c8216 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -5,24 +5,13 @@
# https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates
# TODO:
-# use tsort and mksrcinfo
-
-set -e
-
-lock_file="/tmp/get-package-updates.lock"
-
-# TODO:
# include repository of package customizations
-declare -A repo_paths
-repo_paths["packages"]="/usr/src/archlinux/packages"
-repo_paths["community"]="/usr/src/archlinux/community"
-
-base_dir="$(dirname "$(readlink -f "$0")")/.."
+. "${0%/*}/../conf/default.conf"
# find the PKGBUILD of a given package in a given repository
# TODO:
-# include repository of package customizations
+# _properly_ include repository of package customizations
find_pkgbuild() {
@@ -31,6 +20,7 @@ find_pkgbuild() {
ls "${prefix}/$1/repos/$2-"*"/PKGBUILD" 2> /dev/null && break
done | \
tr ' ' '\n' | \
+ grep -v -- '-i686/PKGBUILD$' | \
grep -v -- '-\(staging\|testing\)-[^/]\+/PKGBUILD$' | \
sort | \
tail -n1
@@ -52,9 +42,16 @@ done
declare -A old_repo_revisions
for repo in "${!repo_paths[@]}"; do
- old_repo_revisions["${repo}"]="$(cat ${base_dir}/work/${repo}.revision || echo NONE)"
+ old_repo_revisions["${repo}"]="$(
+ cat "${work_dir}/${repo}.revision" 2> /dev/null || \
+ echo NONE
+ )"
+ git -C "${repo_paths["${repo}"]}" rev-parse HEAD > \
+ "${work_dir}/${repo}.revision.new"
done
+echo 'Check modified packages from the last update, and put them to the build list.'
+
# Check modified packages from the last update, and put them to the build list.
# If a package is updated, but already on the rebuild list, then just update the git revision number.
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
@@ -63,38 +60,43 @@ done
# TODO:
# ignore packages from the blacklist
-for repo in "${!repo_paths[@]}"; do
- current_HEAD="$(
- git -C "${repo_paths["${repo}"]}" rev-parse HEAD
- )"
- (
- if [ "${old_repo_revisions["${repo}"]}" == "NONE" ]; then
- git -C "${repo_paths["${repo}"]}" archive --format=tar HEAD | \
- tar -t | \
- sed 's|^|A\t|'
- else
- git -C "${repo_paths["${repo}"]}" diff --name-status "${old_repo_revisions["${repo}"]}" HEAD
- fi
- ) | \
- grep '^.\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
- sed 's|^\(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2 '"${current_HEAD}"' \3|' | \
- grep -v '\(staging\|testing\)$'
-done | \
- sort -u | \
+cp "${work_dir}/build-list"{,.new}
+cp "${work_dir}/deletion-list"{,.new}
+
+(
+ for repo in "${!repo_paths[@]}"; do
+ current_HEAD="$(cat "${work_dir}/${repo}.revision.new")"
+ (
+ if [ "${old_repo_revisions["${repo}"]}" == "NONE" ]; then
+ git -C "${repo_paths["${repo}"]}" archive --format=tar HEAD | \
+ tar -t | \
+ sed 's|^|A\t|'
+ else
+ git -C "${repo_paths["${repo}"]}" diff --name-status "${old_repo_revisions["${repo}"]}" HEAD
+ fi
+ ) | \
+ grep '^.\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
+ grep -v -- '-i686/PKGBUILD$' | \
+ sed 's|^\(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2 '"${current_HEAD}"' \3|' | \
+ grep -v '\(staging\|testing\)$'
+ done | \
+ sort -u
+ sed "s|^\(.*\)\$|D \1 $(cat "${work_dir}/archlinux32.revision.new") archlinux32|" "${repo_paths["archlinux32"]}/blacklist"
+) | \
while read -r mode package git_revision repository; do
case "${mode}" in
"A"|"M")
- sed -i "/^${package} /d" "${base_dir}/work/build-list"
+ sed -i "/^${package} /d" "${work_dir}/build-list"
echo "${package} ${git_revision} ${repository}" >> \
- "${base_dir}/work/build-list"
- sed -i "/^${package}\$/d" "${base_dir}/work/deletion-list"
+ "${work_dir}/build-list.new"
+ sed -i "/^${package}\$/d" "${work_dir}/deletion-list.new"
;;
"D")
echo "${package}" >> \
- "${base_dir}/work/deletion-list"
- sed -i "/^${package} /d" "${base_dir}/work/build-list"
+ "${work_dir}/deletion-list.new"
+ sed -i "/^${package} /d" "${work_dir}/build-list.new"
;;
*)
@@ -105,176 +107,106 @@ done | \
esac
done
+echo 'Extract dependencies of packages.'
+
# Put the list in the proper build order.
+# First, we extract the dependencies of each package.
+# TODO:
+# package modifications need to be included here
+
+mkdir -p "${work_dir}/package-infos"
-declare -A packages_built
-declare -A packages_needed
-declare -A packages_needed_for_make
+rm -f "${work_dir}/build-order"
-while read -r package dummy repository; do
+while read -r package git_rev repository; do
PKGBUILD="$(find_pkgbuild "${package}" "${repository}")"
if [ ! -r "${PKGBUILD}" ]; then
- echo "'${package}'" "'${repository}'" "'${PKGBUILD}'"
+ echo "can't find PKGBUILD to package '${package}' from repository '${repository}': '${PKGBUILD}'"
exit 1
fi
- content="$(
- sed 's|#.*$||;s|^\s\+||;s|\s\+$||;/^$/d' "${PKGBUILD}" | \
- sed '
- :begin;
- $!N;
- s@^\([^=]\+=([^)]*\)\n\(.*\)$@\1 \2@;
- tbegin;
- P;
- D
- '
- )"
-
- packages_built["${package}"]="$(
- echo "${content}" | \
- grep '^provides=\|^pkgname=' | \
- sed 's|^[^=]\+=||; s|=.*$||' | \
- tr "' ()\"" '\n' | \
- sort -u | \
- sed '/^$/d'
- )"
+ if [ ! -e "${work_dir}/package-infos/${package}.${git_rev}.builds" ] || \
+ [ ! -e "${work_dir}/package-infos/${package}.${git_rev}.needs" ]; then
- packages_needed["${package}"]="$(
(
- echo "${content}" | \
- grep '^depends=' | \
- sed 's|^[^=]\+=||; s|>\?=.*$||' | \
- tr "' ()\"" '\n' | \
- sort -u | \
- sed '/^$/d'
- echo "${packages_built["${package}"]}" | \
- sed 'p'
- ) | \
- sort | \
- uniq -u
- )"
+ cd "${PKGBUILD%/*}"
+ mksrcinfo -o "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO"
+ )
+
+ grep '^\('$'\t''provides\|pkgname\) = ' "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO" | \
+ cut -d= -f2 | \
+ sed 's|^\s\+||; s|[<>]$||' | \
+ sort -u > \
+ "${work_dir}/package-infos/${package}.${git_rev}.builds"
- packages_needed_for_make["${package}"]="$(
(
- echo "${content}" | \
- grep '^makedepends=' | \
- sed 's|^[^=]\+=||; s|>\?=.*$||' | \
- tr "' ()\"" '\n' | \
- sort -u | \
- sed '/^$/d'
- echo "${packages_built["${package}"]}" | \
+ (
+ sed -n '/^pkgname = /q;/^'$'\t''depends = /p' "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO"
+ grep '^'$'\t''\(makedepends\|checkdepends\) = ' "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO"
+ ) | \
+ cut -d= -f2 | \
+ sed 's|^\s\+||; s|[<>]$||' | \
+ sort -u
+ cat "${work_dir}/package-infos/${package}.${git_rev}.builds" | \
sed 'p'
) | \
sort | \
- uniq -u
- )"
-
-done < "${base_dir}/work/build-list"
-
-rm -f "${base_dir}/work/build-list.new"
+ uniq -u > \
+ "${work_dir}/package-infos/${package}.${git_rev}.needs"
-while [ ${#packages_built[@]} -gt 0 ]; do
+ rm "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO"
- found_something=false
-
- for package in "${!packages_built[@]}"; do
-
- if [ -z "$(
- (
- echo "${packages_needed["${package}"]}"
- echo "${packages_needed_for_make["${package}"]}"
- echo "${!packages_built[@]}" | \
- tr ' ' '\n'
- ) | \
- sort | \
- uniq -d
- )" ]; then
- grep "^${package} " "${base_dir}/work/build-list" >> \
- "${base_dir}/work/build-list.new"
- unset packages_built["${package}"]
- unset packages_needed["${package}"]
- unset packages_needed_for_make["${package}"]
- found_something=true
- fi
- done
-
- if ${found_something}; then
- continue
- fi
-
- for package in "${!packages_built[@]}"; do
-
- if [ -z "$(
- (
- echo "${packages_needed_for_make["${package}"]}"
- echo "${!packages_built[@]}" | \
- tr ' ' '\n'
- ) | \
- sort | \
- uniq -d
- )" ]; then
- grep "^${package} " "${base_dir}/work/build-list" >> \
- "${base_dir}/work/build-list.new"
- unset packages_built["${package}"]
- unset packages_needed["${package}"]
- unset packages_needed_for_make["${package}"]
- found_something=true
- break
- fi
- done
-
- if ${found_something}; then
- continue
- fi
-
- for package in "${!packages_built[@]}"; do
-
- if [ -z "$(
- (
- echo "${packages_needed["${package}"]}"
- echo "${!packages_built[@]}" | \
- tr ' ' '\n'
- ) | \
- sort | \
- uniq -d
- )" ]; then
- grep "^${package} " "${base_dir}/work/build-list" >> \
- "${base_dir}/work/build-list.new"
- unset packages_built["${package}"]
- unset packages_needed["${package}"]
- unset packages_needed_for_make["${package}"]
- found_something=true
- break
- fi
- done
-
- if ${found_something}; then
- continue
fi
- >&2 echo 'uaagh: there seem to be cyclic dependencies - I give up.'
- >&2 echo "packages left: ${!packages_built[@]}"
-
- for pkg in "${!packages_built[@]}"; do
- echo "${pkg}:"
- echo ${packages_needed["${pkg}"]}
+ while read target; do
+ echo "${target} ${target}" >> \
+ "${work_dir}/build-order"
+ while read dependency; do
+ echo "${target} ${dependency}" >> \
+ "${work_dir}/build-order"
+ done < "${work_dir}/package-infos/${package}.${git_rev}.needs"
+ done < "${work_dir}/package-infos/${package}.${git_rev}.builds"
+
+done < "${work_dir}/build-list.new"
+
+echo 'Now actually sort it.'
+
+(
+ tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \
+ nl -ba | \
+ awk '{print $1 " not-git whatever " $2}'
+ awk '{print "0 " $2 " " $3 " " $1}' < \
+ "${work_dir}/build-list.new"
+) | \
+ sort -k4,4 -k1nr | \
+ uniq -f3 -D | \
+ sed 'N;s|\n| |' | \
+ sort -k1n,1 | \
+ awk '{print $4 " " $6 " " $7}' > \
+ "${work_dir}/build-list.new.new"
+
+if [ -s "${work_dir}/tsort.error" ]; then
+ >&2 echo 'WARNING: There is a dependency cycle!'
+ >&2 cat "${work_dir}/tsort.error"
+ >&2 echo
+ >&2 echo 'I will continue anyway.'
+else
+ rm "${work_dir}/tsort.error"
+fi
+
+# Move the .new-files to the actual files
+
+(
+ echo "build-list.new" "build-list" "deletion-list"
+ echo "${!repo_paths[@]}" | \
+ sed 's@\( \|$\)@.revision\1@g'
+) | \
+ tr ' ' '\n' | \
+ while read -r file; do
+ mv "${work_dir}/${file}.new" "${work_dir}/${file}"
done
-# TODO: Break circular dependencies
-
- exit 1
-done
-
-mv "${base_dir}/work/build-list.new" "${base_dir}/work/build-list"
-
-# Write the current git revision numbers to files.
-
-for repo in "${!repo_paths[@]}"; do
- git -C "${repo_paths["${repo}"]}" rev-parse HEAD > \
- "${base_dir}/work/${repo}.revision"
-done
-
# Remove the lock file
rm -f "${lock_file}"
diff --git a/conf/.gitignore b/conf/.gitignore
new file mode 100644
index 0000000..f21e1b2
--- /dev/null
+++ b/conf/.gitignore
@@ -0,0 +1 @@
+local.conf
diff --git a/conf/default.conf b/conf/default.conf
new file mode 100755
index 0000000..1517272
--- /dev/null
+++ b/conf/default.conf
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# set up some common variables
+
+set -e
+
+base_dir="$(dirname "$(readlink -f "$0")")/.."
+
+work_dir="${base_dir}/work"
+
+declare -A repo_paths
+repo_paths["packages"]="${work_dir}/repos/packages"
+repo_paths["community"]="${work_dir}/repos/community"
+repo_paths["archlinux32"]="${work_dir}/repos/packages32"
+
+lock_file="/tmp/${0##*/}.lock"
+
+# possibly pull in custom modifications
+
+[ -r "${base_dir}/conf/local.conf" ] && . "${base_dir}/conf/local.conf"
+
+# check / set up environment
+
+mkdir -p "${work_dir}"
+touch "${work_dir}/build-list"
+touch "${work_dir}/deletion-list"
+
+for repo in "${!repo_paths[@]}"; do
+
+ mkdir -p "${repo_paths["${repo}"]%/*}"
+
+ if ! git -C "${repo_paths["${repo}"]}" status &> /dev/null; then
+ if [ "${repo}" == "packages32" ]; then
+ repo_source='git@github.com:archlinux32/packages.git'
+ else
+ repo_source="git://git.archlinux.org/svntogit/${repo}.git"
+ fi
+ git clone "${repo_source}" "${repo_paths["${repo}"]}"
+ fi
+
+done