summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2018-03-22 15:19:23 +0100
committerErich Eckner <git@eckner.net>2018-03-22 15:19:23 +0100
commitc772be57eca3a2176ac3931c02fb582f85c0d575 (patch)
tree276bbc6566e141444d37ee464a0c9c6c0ac956fe
parente808f93c3f9af51fa32764d3195bbba33b73d026 (diff)
downloadbuilder-c772be57eca3a2176ac3931c02fb582f85c0d575.tar.xz
decommission package-states directory - we rely on the database now
-rwxr-xr-xbin/bootstrap-mysql60
-rwxr-xr-xbin/cleanup20
-rwxr-xr-xbin/db-update62
-rwxr-xr-xbin/get-assignment27
-rwxr-xr-xbin/get-package-updates6
-rwxr-xr-xbin/modify-package-state38
-rwxr-xr-xbin/return-assignment38
-rwxr-xr-xlib/common-functions4
8 files changed, 7 insertions, 248 deletions
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 4c1938f..fc347d7 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -3,6 +3,10 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
+# TODO: this should become some sort of mysql dump to bootstrap the
+# tables and stored functions only - when the database is gone, we're
+# pretty much screwed anyway :-D
+
tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir)
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
@@ -792,68 +796,12 @@ fi
} | \
mysql_run_query
-find "${work_dir}/package-states" \
- -maxdepth 1 \
- -mindepth 1 \
- \( \
- -name '*.done' -o \
- -name '*.testing' -o \
- -name '*.tested' \
- \) \
- -exec sed '
- 1!d
- s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@
- s@^\S*/@@
- s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 /
- s/ \([0-9]\+\) / \1.0 /
- s/ [0-9]\+\.\([0-9]\+\) / \1 /
- ' {} \; | \
- while read -r state_file sub_pkgrel arch package_file; do
- printf '%s ' "${state_file}"
- repository=$(
- grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \
- cut -d' ' -f1
- )
- if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then
- printf 'not found exactly once on the master mirror.\n'
- >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}"
- continue
- fi
- mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}"
- printf ' ok\n'
- done
-
tr ' ' '.' < \
"${work_dir}/build-list" | \
while read -r state_file; do
mysql_generate_package_metadata 'build-list' "${state_file}"
done
-# shellcheck disable=SC2016
-find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \
- sed '
- s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3|
- ' | \
- while read -r state_file pkgbase git_revision mod_git_revision repository; do
- printf 'UPDATE `build_assignments`'
- mysql_join_build_assignments_package_sources
- mysql_join_package_sources_upstream_repositories
- printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \
- "$(
- tr -d '\n' < \
- "${state_file}" | \
- base64 -w0
- )"
- printf ' WHERE'
- printf ' `package_sources`.`%s`=from_base64("%s") AND' \
- 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \
- 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \
- 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)"
- printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
- "$(printf '%s' "${repository}" | base64 -w0)"
- done | \
- mysql_run_query
-
grep '^\('"$(
# shellcheck disable=SC2086
printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories}
diff --git a/bin/cleanup b/bin/cleanup
index a0bf790..7aaa39f 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -19,26 +19,6 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
exit
fi
-# remove blocked/broken/locked markes of packages not on the buildlist anymore
-
-{
- find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \
- sed '
- s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5|
- '
- awk '{
- print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken"
- print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked"
- print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked"
- }' "${work_dir}/build-list"
-} | \
- sort -k3 | \
- uniq -uf2 | \
- grep '^state ' | \
- awk '{print $2}' | \
- sed "s|^|${work_dir}/package-states/|" | \
- xargs -rn1 rm
-
# remove logs where package is not broken/locked anymore
{
diff --git a/bin/db-update b/bin/db-update
index 6ecf9c9..6722f24 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -207,12 +207,6 @@ for source_stability in 'testing' 'staging'; do
mysql_join_repositories_repository_stabilities
printf ' AND `repository_stabilities`.`name`="%s"' \
"${source_stability}"
- # TODO: we may relax this condition, if we a) got rid of state files
- # or b) improved keeping them in sync (*.testing -> /dev/null is not
- # implemented, only .tested -> /dev/null)
- if [ ! "${source_stability}" = 'staging' ]; then
- printf ' AND `binary_packages`.`is_tested`'
- fi
mysql_join_binary_packages_build_assignments
mysql_join_build_assignments_package_sources
mysql_join_package_sources_upstream_repositories
@@ -401,62 +395,6 @@ for source_stability in 'testing' 'staging'; do
printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n'
} | \
mysql_run_query
-
- sed_apply_rm=$(
- while read -r pkg _ _; do
- printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
- done < \
- "${tmp_dir}/rm"
- )
-
- sed_apply_remove_part_of_mv=$(
- while read -r pkg _ _; do
- printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
- done < \
- "${tmp_dir}/mv"
- )
-
- sed_apply_move_part_of_mv=$(
- while read -r pkg _ _; do
- printf '/^%s$/{\n' "$(str_to_regex "${pkg}")"
- printf 'w %%s\n'
- printf 'd\n'
- printf '}\n'
- done < \
- "${tmp_dir}/mv"
- )
-
- find "${work_dir}/package-states/" \
- \( \
- -name '*.done' -o \
- -name '*.testing' -o \
- -name '*.tested' \
- \) \
- -exec sed -i "${sed_apply_rm}" '{}' \;
- find "${work_dir}/package-states/" \
- -name '*.tested' \
- -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \;
- find "${work_dir}/package-states/" \
- -name '*.done' \
- -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
- -printf '%p\n' | \
- while read -r sf; do
- sed -i "$(
- printf '%s\n' "${sed_apply_move_part_of_mv}" | \
- sed '
- s,%s,'"${sf%.done}.testing"',g
- '
- )" "${sf}"
- done
- find "${work_dir}/package-states/" \
- \( \
- -name '*.done' -o \
- -name '*.testing' -o \
- -name '*.tested' \
- \) \
- -type f \
- -not -exec test -s '{}' \; \
- -delete
done
trigger_mirror_refreshs
diff --git a/bin/get-assignment b/bin/get-assignment
index 3357fc6..a82e1f8 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -18,17 +18,8 @@
# TODO: honor manual build order of tool-chain:
# toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc
-mkdir -p "${work_dir}/package-states"
-
hand_out_assignment() {
- # we don't care anymore if an older version of this package was
- # "locked" or "broken"
- find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
- -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \
- -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \
- -delete
-
# move that build order to the end of the build-list
sed -i '
/^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
@@ -57,21 +48,7 @@ hand_out_assignment() {
printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
"$(printf '%s' "$4" | base64 -w0)"
printf ' LIMIT 1;\n'
- } | \
- mysql_run_query | \
- tr '\t' ' '
- {
- # shellcheck disable=SC2154
- echo "${slave}"
- if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
- cat "${work_dir}/package-states/$1.$2.$3.$4.locked"
- fi
- } | \
- sort -u | \
- sponge "${work_dir}/package-states/$1.$2.$3.$4.locked"
- # shellcheck disable=SC2016
- {
printf 'UPDATE `build_slaves`'
printf ' SET `currently_building` = ('
printf ' SELECT `build_assignments`.`id`'
@@ -86,6 +63,7 @@ hand_out_assignment() {
printf ' `upstream_repositories`.`name` = from_base64("%s")' \
"$(printf '%s' "$4" | base64 -w0)"
printf ')'
+ # shellcheck disable=SC2154
printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
"$(printf '%s' "${slave}" | base64 -w0)"
@@ -101,7 +79,8 @@ hand_out_assignment() {
printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \
"$(printf '%s' "$4" | base64 -w0)"
} | \
- mysql_run_query
+ mysql_run_query | \
+ tr '\t' ' '
exit 0
diff --git a/bin/get-package-updates b/bin/get-package-updates
index a36cb8a..3e9edf9 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -111,9 +111,6 @@ delete_package() {
mysql_run_query >> \
"${work_dir}/deletion-list.new"
sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
- find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \
- -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \
- -delete
# TODO: Once we want to rely on the database for test_exclusion, we
# need to run the command below unconditionally, but with some
# changes, so we can easily revert.
@@ -317,9 +314,6 @@ done | \
if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
# new or modified PKGBUILD
sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
- find "${work_dir}/package-states" -regextype grep \
- -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \
- -delete
# shellcheck disable=SC2154
echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
"${work_dir}/build-list.new"
diff --git a/bin/modify-package-state b/bin/modify-package-state
index 76b8c1e..4a2f5ac 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -243,46 +243,8 @@ while read -r package reason; do
{
printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}"
printf ';\n'
- # TODO: remove the below part, once the database if fully functional
- printf 'UPDATE `binary_packages` AS `a`'
- printf ' JOIN `binary_packages` AS `b`'
- printf ' ON `a`.`build_assignment`=`b`.`build_assignment`'
- mysql_join_binary_packages_repositories 'b'
- printf ' SET `b`.`is_tested`=1'
- printf ' WHERE `a`.`is_tested`'
- printf ' AND NOT `b`.`is_tested`'
- printf ' AND NOT `repositories`.`name`="deletion-list";\n'
} | \
mysql_run_query
- case "${action}" in
- 'block')
- printf '%s' "${reason}" > \
- "${work_dir}/package-states/${package}.blocked"
- ;;
- 'unblock')
- rm -f "${work_dir}/package-states/${package}.blocked"
- ;;
- 'faulty')
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
- -name '*.tested' \
- -exec grep -lxF "${package}" '{}' \; | \
- while read -r sf; do
- mv "${sf}" "${sf%ed}ing"
- done
- ;;
- 'tested')
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
- -name '*.testing' \
- -exec grep -lxF "${package}" '{}' \; | \
- while read -r sf; do
- mv "${sf}" "${sf%ing}ed"
- done
- ;;
- *)
- >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
- exit 42
- ;;
- esac
printf '%s %s\n' "${action}" "${package}"
fi
done \
diff --git a/bin/return-assignment b/bin/return-assignment
index 02bd491..da9e762 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -89,9 +89,6 @@ if [ "$5" = 'ERROR' ]; then
'*.build-log.gz'
)
- # shellcheck disable=SC2154
- echo "${slave}" >> \
- "${work_dir}/package-states/$1.$2.$3.$4.broken"
# shellcheck disable=SC2016
{
if [ -n "${saved_build_logs}" ]; then
@@ -162,23 +159,6 @@ if [ "$5" = 'ERROR' ]; then
} | \
mysql_run_query
- # shellcheck disable=SC2154
- sed -i '
- /^'"$(str_to_regex "${slave}")"'$/d
- ' "${work_dir}/package-states/$1.$2.$3.$4.locked"
- if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
- rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
-
- # move that build order to the end of the build-list
- sed -i '
- /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
- $ b
- d
- }
- $ a '"$1 $2 $3 $4" \
- "${work_dir}/build-list"
- fi
-
# release lock on build-list - otherwise seed-build-list won't run
flock -u 9
@@ -517,24 +497,9 @@ for package in ${packages}; do
remove_old_package_versions 'i686' "${destination}" "${package}"
done
-# remove old state files (these should be only "done" markers, but
-# actually we don't care what it is) - as long as it's not "testing" or "tested"
-find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
- -not -name '*.testing' \
- -not -name '*.tested' \
- -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \
- -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \
- -exec rm '{}' \;
-
# remove package from build list
sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
-# remove package lock file
-if ! [ "${destination}" = 'build-support' ]; then
- # shellcheck disable=SC2086
- printf '%s\n' ${packages} > \
- "${work_dir}/package-states/$1.$2.$3.$4.done"
-fi
# shellcheck disable=SC2016
{
printf 'UPDATE `build_assignments`'
@@ -578,6 +543,3 @@ fi
printf 'DROP TABLE `loops_to_delete`;\n'
} | \
mysql_run_query
-rm -f \
- "${work_dir}/package-states/$1.$2.$3.$4.locked" \
- "${work_dir}/package-states/$1.$2.$3.$4.broken"
diff --git a/lib/common-functions b/lib/common-functions
index 0a5ef1c..f66ccd1 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -4,10 +4,6 @@
# shellcheck disable=SC2039
-# TODO: have full information (currently in files) in database
-
-# TODO: remove state files
-
if [ -z "${base_dir}" ]; then
# just to make shellcheck happy
. 'conf/default.conf'