summaryrefslogtreecommitdiff
path: root/bin
diff options
context:
space:
mode:
authorAndreas Baumann <mail@andreasbaumann.cc>2018-03-23 20:18:01 +0100
committerAndreas Baumann <mail@andreasbaumann.cc>2018-03-23 20:18:01 +0100
commitc30de005f885202f24929bd4e3d3f5c885efbc0a (patch)
tree44b512356b80d3adad6521ad74f38ff9271f6c0d /bin
parentff768f012bfef1bf264d06214aead70a58c0ff90 (diff)
parent497779257683e1c4ee2f2bf4c25687b34323c6be (diff)
downloadbuilder-c30de005f885202f24929bd4e3d3f5c885efbc0a.tar.xz
Merge branch 'master' into opcodes
Diffstat (limited to 'bin')
-rwxr-xr-xbin/bootstrap-mysql522
-rwxr-xr-xbin/build-master-status695
-rwxr-xr-xbin/build-master-status-from-mysql152
-rwxr-xr-xbin/build-packages126
-rwxr-xr-xbin/calculate-dependent-packages184
-rwxr-xr-xbin/check-bugtracker79
-rwxr-xr-xbin/check-opcodes2
-rwxr-xr-xbin/cleanup70
-rwxr-xr-xbin/common-functions1348
-rwxr-xr-xbin/copy-to-build-support187
-rwxr-xr-xbin/db-update875
-rwxr-xr-xbin/delete-packages407
-rwxr-xr-xbin/get-assignment306
-rwxr-xr-xbin/get-package-updates566
-rwxr-xr-xbin/ii-connect13
-rwxr-xr-xbin/interpret-mail34
-rwxr-xr-xbin/modify-package-state212
-rwxr-xr-xbin/mysql-functions647
-rwxr-xr-xbin/ping-from-slave25
-rwxr-xr-xbin/ping-to-master39
-rwxr-xr-xbin/prioritize-build-list19
-rwxr-xr-xbin/repo-copy92
-rwxr-xr-xbin/return-assignment342
-rwxr-xr-xbin/sanity-check186
-rwxr-xr-xbin/seed-build-list455
-rwxr-xr-xbin/show-dependencies638
-rwxr-xr-xbin/slave-build-connect9
-rwxr-xr-xbin/strict-bashism-check39
-rwxr-xr-xbin/why-dont-you465
29 files changed, 3247 insertions, 5487 deletions
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 46678f7..86fa645 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -3,6 +3,10 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
+# TODO: this should become some sort of mysql dump to bootstrap the
+# tables and stored functions only - when the database is gone, we're
+# pretty much screwed anyway :-D
+
tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir)
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
@@ -32,6 +36,8 @@ done | \
sort -k2,2 > \
"${tmp_dir}/master-mirror-listing"
+# TODO: include hash of binary package in mysql database
+
if [ ! "$1" = 'slim' ]; then
tables=$(
printf '%s\n' \
@@ -61,6 +67,7 @@ if [ ! "$1" = 'slim' ]; then
' sub_pkgrel MEDIUMINT' \
' has_issues BIT' \
' is_tested BIT' \
+ ' is_to_be_deleted BIT' \
' pkgname VARCHAR(64)' \
' architecture SMALLINT :architectures' \
' UNIQUE content build_assignment sub_pkgrel pkgname architecture repository' \
@@ -81,7 +88,9 @@ if [ ! "$1" = 'slim' ]; then
' package_source BIGINT :package_sources' \
' architecture SMALLINT :architectures' \
' is_blocked VARCHAR(128) NULL' \
+ ' is_black_listed TEXT NULL' \
' is_broken BIT' \
+ ' priority SMALLINT' \
' UNIQUE content package_source architecture' \
'build_dependency_loops BIGINT' \
' loop MEDIUMINT' \
@@ -99,6 +108,8 @@ if [ ! "$1" = 'slim' ]; then
' git_revision VARCHAR(40)' \
' mod_git_revision VARCHAR(40)' \
' upstream_package_repository SMALLINT :upstream_repositories' \
+ ' uses_upstream BIT' \
+ ' uses_modification BIT' \
' UNIQUE content pkgbase git_revision mod_git_revision' \
'upstream_repositories SMALLINT' \
' name VARCHAR(64)' \
@@ -122,9 +133,40 @@ if [ ! "$1" = 'slim' ]; then
' build_assignment BIGINT :build_assignments' \
' date DATETIME' \
' reason SMALLINT :fail_reasons' \
- ' log_file VARCHAR(512)'
+ ' log_file VARCHAR(512)' \
+ 'todos MEDIUMINT' \
+ ' file VARCHAR(64)' \
+ ' line MEDIUMINT' \
+ ' description VARCHAR(512)'
+ 'todo_links MEDIUMINT' \
+ ' dependent MEDIUMINT' \
+ ' depending_on MEDIUMINT' \
+ 'repository_stability_relations MEDIUMINT' \
+ ' more_stable MEDIUMINT :repository_stabilities' \
+ ' less_stable MEDIUMINT :repository_stabilities' \
+ 'repository_moves MEDIUMINT' \
+ ' from_repository MEDIUMINT :repositories' \
+ ' to_repository MEDIUMINT :repositories' \
+ ' upstream_package_repository SMALLINT :upstream_repositories' \
+ ' UNIQUE source from_repository upstream_package_repository' \
+ 'statistics BIGINT' \
+ ' date DATETIME' \
+ ' stable_packages_count MEDIUMINT' \
+ ' pending_tasks_count MEDIUMINT' \
+ ' pending_packages_count MEDIUMINT' \
+ ' staging_packages_count MEDIUMINT' \
+ ' testing_packages_count MEDIUMINT' \
+ ' tested_packages_count MEDIUMINT' \
+ ' broken_tasks_count MEDIUMINT' \
+ ' dependency_loops_count MEDIUMINT' \
+ ' dependency_looped_tasks_count MEDIUMINT' \
+ ' locked_tasks_count MEDIUMINT' \
+ ' blocked_tasks_count MEDIUMINT' \
+ ' next_tasks_count MEDIUMINT' \
+ ' UNIQUE date'
)
+ # shellcheck disable=SC2016
{
printf '%s\n' \
'/*!40014 SET UNIQUE_CHECKS=0 */;' \
@@ -137,14 +179,11 @@ if [ ! "$1" = 'slim' ]; then
while read -r table size; do
case "${turn}" in
'drop')
- # shellcheck disable=SC2016
printf 'DROP TABLE IF EXISTS `%s`;\n' "${table}"
;;
'create'|'link')
if [ "${turn}" = 'create' ]; then
- # shellcheck disable=SC2016
printf 'CREATE TABLE `%s` (\n' "${table}"
- # shellcheck disable=SC2016
printf ' `id` %s NOT NULL AUTO_INCREMENT,\n' "${size}"
fi
echo "${tables}" | \
@@ -167,16 +206,14 @@ if [ ! "$1" = 'slim' ]; then
if [ -z "${rest}" ]; then
rest="${type}"
fi
- # shellcheck disable=SC2016
printf ' UNIQUE KEY `%s` (' "${type}"
- # shellcheck disable=SC2016,SC2086
+ # shellcheck disable=SC2086
printf '`%s`,' ${rest} | \
sed 's|,$||'
printf ')'
;;
*)
if [ "${turn}" = 'create' ]; then
- # shellcheck disable=SC2016
printf ' `%s` %s' "${column}" "${type}"
if ! echo "${rest}" | \
grep -qwF 'NULL'; then
@@ -187,7 +224,6 @@ if [ ! "$1" = 'slim' ]; then
tr ' ' '\n' | \
sed -n 's/^://;T;p' | \
while read -r link; do
- # shellcheck disable=SC2016
printf 'ALTER TABLE `%s` ADD FOREIGN KEY (`%s`) REFERENCES `%s` (`id`);\n' \
"${table}" \
"${column}" \
@@ -201,7 +237,6 @@ if [ ! "$1" = 'slim' ]; then
fi
done
if [ "${turn}" = 'create' ]; then
- # shellcheck disable=SC2016
printf ' PRIMARY KEY (`id`));\n'
fi
;;
@@ -214,28 +249,318 @@ if [ ! "$1" = 'slim' ]; then
printf '%s\n' \
'/*!40014 SET UNIQUE_CHECKS=1 */;' \
'/*!40014 SET FOREIGN_KEY_CHECKS=1 */;'
+
+ # show_broken_packages_and_dependencies
+ # gives a list of broken packages and their dependencies if they're
+ # still on the build- or deletion-list
+ printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n'
+ printf 'DELIMITER //\n'
+ printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n'
+ printf 'BEGIN\n'
+ printf 'CREATE TEMPORARY TABLE `%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \
+ 'broken_packages_and_dependencies' 'broken_packages_and_dependencies_old'
+ printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)'
+ printf ' SELECT `build_assignments`.`id`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_build_assignments
+ printf ' AND `build_assignments`.`is_broken`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`name`="build-list";\n'
+ printf 'REPEAT\n'
+ printf 'INSERT IGNORE INTO `broken_packages_and_dependencies_old` (`id`)'
+ printf ' SELECT `broken_packages_and_dependencies`.`id` FROM `broken_packages_and_dependencies`;\n'
+ printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)'
+ printf ' SELECT `new_bp`.`build_assignment`'
+ printf ' FROM `broken_packages_and_dependencies_old`'
+ mysql_join_build_assignments_binary_packages 'broken_packages_and_dependencies_old' 'old_bp'
+ mysql_join_binary_packages_dependencies 'old_bp'
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_building`'
+ mysql_join_dependencies_install_target_providers
+ mysql_join_install_target_providers_binary_packages '' 'new_bp'
+ mysql_join_binary_packages_repositories 'new_bp' 'new_repo'
+ printf ' AND `new_repo`.`name` IN ("build-list","deletion-list");\n'
+ printf 'UNTIL ROW_COUNT()=0\n'
+ printf 'END REPEAT;\n'
+ printf 'SELECT '
+ mysql_query_select_pkgbase_and_revision
+ printf ' JOIN `broken_packages_and_dependencies` ON `broken_packages_and_dependencies`.`id`=`build_assignments`.`id`;\n'
+ printf 'END\n'
+ printf '//\n'
+ printf 'DELIMITER ;\n'
+
+ # calculate_dependencies_of_package_upto_first_built_one
+ # save binary packages (only their `id`) in `relevant_binary_packages` and `relevant_binary_packages_copy`
+ printf 'DROP PROCEDURE IF EXISTS calculate_dependencies_of_package_upto_first_built_one;\n'
+ printf 'DELIMITER //\n'
+ printf 'CREATE PROCEDURE calculate_dependencies_of_package_upto_first_built_one(IN `target_pkgbase` VARCHAR(64))\n'
+ printf 'BEGIN\n'
+ printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)'
+ printf ' SELECT `binary_packages`.`id`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`name`="build-list"'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ printf ' WHERE `package_sources`.`pkgbase`=`target_pkgbase`;\n'
+ printf 'REPEAT\n'
+ printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)'
+ printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`;\n'
+ printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)'
+ printf ' SELECT `install_target_providers`.`package`'
+ printf ' FROM `relevant_binary_packages_copy`'
+ printf ' JOIN `binary_packages` ON `relevant_binary_packages_copy`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`name`="build-list"'
+ mysql_join_binary_packages_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_building`'
+ mysql_join_dependencies_install_target_providers
+ printf ';\n'
+ printf 'UNTIL ROW_COUNT()=0\n'
+ printf 'END REPEAT;\n'
+ printf 'END\n'
+ printf '//\n'
+ printf 'DELIMITER ;\n'
+
+ # calculate_maximal_moveable_set
+ # stores results in `moveable_binary_packages` and `replaced_binary_packages`
+ # Give a maximal list of packages to be moved, while implementing the
+ # condition from db-update:
+
+ # Every package which is replaced[1], must have its provided install_targets:
+ # a) provided by another moved or not-replaced package or
+ # b) not required by any not-replaced package.
+
+ # Every package being moved needs to have all dependencies
+ # installable in the target repository.
+
+ # TODO: [1] A "replaced" package may also be in a different repository
+ # e.g. if a-2 is moved from [staging] to [testing] and there is only
+ # a-1 in [core], then this will be "replaced" by a-2 on a system
+ # running on [testing] repositories.
+
+ printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n'
+ printf 'DELIMITER //\n'
+ printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n'
+ printf 'BEGIN\n'
+ # variables to store count of changed rows
+ printf 'DECLARE row_count_saved INT DEFAULT 0;\n'
+ for copy in '' '_copy' '_copy2'; do
+ printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \
+ 'moveable' "${copy}" \
+ 'replaced' "${copy}"
+ printf 'CREATE TEMPORARY TABLE `replaced_binary_packages%s` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' \
+ "${copy}"
+ if [ "${copy}" = '_copy2' ]; then
+ continue
+ fi
+ printf 'CREATE TEMPORARY TABLE `moveable_binary_packages%s` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' \
+ "${copy}"
+ done
+ # these packages are considered for moving:
+ printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+ printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
+ # correct stability: "testing"/"staging" - as chosen
+ printf ' WHERE `repository_stabilities`.`name` = `from_stability`'
+ printf ' AND ('
+ printf '`from_stability`="staging"'
+ # "testing" packages must be tested
+ printf ' OR `binary_packages`.`is_tested`'
+ printf ')'
+ # no open issues
+ printf ' AND NOT `binary_packages`.`has_issues`'
+ printf ';\n'
+ # these packages are considered for being replaced:
+ # for each moved package
+ printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+ printf ' SELECT `r_bp`.`id`,`m_bp`.`id`'
+ printf ' FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`'
+ mysql_join_binary_packages_repositories 'm_bp' 'm_r'
+ mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba'
+ mysql_join_build_assignments_package_sources 'm_ba' 'm_ps'
+ mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur'
+ mysql_join_upstream_repositories_repository_moves 'm_ur'
+ printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`'
+ # in its target repository
+ printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`'
+ mysql_join_repositories_binary_packages 'r_r' 'r_bp'
+ # all packages with identical names
+ printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`;\n'
+ # now we delete all unreplaceable and unmoveable packages from the respective
+ # list until no further changes are required
+ printf 'REPEAT\n'
+ # create copies of our temporary tables *yuck*
+ for table in 'replaced' 'moveable'; do
+ for copy in '' '2'; do
+ if [ "${table}" = 'moveable' ] && \
+ [ "${copy}" = '2' ]; then
+ continue
+ fi
+ printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}"
+ printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}"
+ printf ' SELECT `%s_binary_packages`.*' "${table}"
+ printf ' FROM `%s_binary_packages`;\n' "${table}"
+ done
+ done
+ # a package is not moveable if its dependencies are not provided ...
+ printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages`'
+ printf ' FROM `replaced_binary_packages`'
+ printf ' RIGHT JOIN `moveable_binary_packages`'
+ printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+ printf ' JOIN `repositories` AS `target_repositories` ON `moveable_binary_packages`.`to_repository`=`target_repositories`.`id`'
+ mysql_join_binary_packages_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ # ... by a not-deleted, "more stable" package already in place or ...
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `install_target_providers`'
+ mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+ mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+ printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`less_stable`'
+ printf ' AND NOT EXISTS ('
+ printf 'SELECT * FROM `replaced_binary_packages_copy`'
+ printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`'
+ printf ')'
+ # ... by a moved package
+ printf ') AND NOT EXISTS ('
+ printf 'SELECT * FROM `install_target_providers`'
+ printf ' JOIN `moveable_binary_packages_copy` ON `moveable_binary_packages_copy`.`id`=`install_target_providers`.`package`'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ');\n'
+ printf 'SET @row_count_saved = ROW_COUNT();\n'
+
+ # refresh copies of our temporary tables *yuck*
+ for table in 'replaced' 'moveable'; do
+ for copy in '' '2'; do
+ if [ "${table}" = 'moveable' ] && \
+ [ "${copy}" = '2' ]; then
+ continue
+ fi
+ printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}"
+ printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}"
+ printf ' SELECT `%s_binary_packages`.*' "${table}"
+ printf ' FROM `%s_binary_packages`;\n' "${table}"
+ done
+ done
+
+ # packages which should not be replaced: ones providing something that is:
+ # a) still needed by a not-replaced package x "less stable" than the target repository and
+ # b) not provided by:
+ # 1) a moved package or
+ # 2) any current, not-replaced package in a repository more stable than x's repository
+ #
+ # Note, that this is not 100% clean from an academic point of view:
+ # It might require _downgrading_ of a package to keep fulfilling a dependency
+ # and it might require to do so _without_ any chance for the user to notice,
+ # because there may be more "dependencies" in the database than in the package files.
+ #
+ # However, in practice both should not happen.
+ #
+ printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`'
+ printf ' JOIN `moveable_binary_packages` ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`'
+ printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`'
+ mysql_join_binary_packages_install_target_providers 'repl_bp'
+ mysql_join_install_target_providers_dependencies
+ printf ' AND NOT EXISTS ('
+ # dependencies of replaced packages don't matter
+ printf 'SELECT * FROM `replaced_binary_packages_copy`'
+ printf ' WHERE `replaced_binary_packages_copy`.`id`=`dependencies`.`dependent`'
+ printf ')'
+ mysql_join_dependencies_dependency_types
+ # consider only runtime dependencies
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ mysql_join_dependencies_binary_packages '' 'req_bp'
+ # we need to check wether req_bp's dependency is (un)critical
+ mysql_join_binary_packages_repositories 'repl_bp' 'repl_r'
+ mysql_join_binary_packages_repositories 'req_bp' 'req_r'
+ # dependent package is "less stable" than dependency
+ printf ' JOIN `repository_stability_relations` AS `repl_rr`'
+ printf ' ON `repl_rr`.`more_stable`=`repl_r`.`stability`'
+ printf ' AND `repl_rr`.`less_stable`=`req_r`.`stability`'
+ # a) ^
+ printf ' WHERE NOT EXISTS ('
+ # no moved package ...
+ printf 'SELECT *'
+ printf ' FROM `moveable_binary_packages_copy`'
+ mysql_join_binary_packages_install_target_providers 'moveable_binary_packages_copy' 'subst_itp'
+ # ... provides the same
+ printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`'
+ # b) 1) ^
+ printf ') AND NOT EXISTS ('
+ # no current package ...
+ printf 'SELECT *'
+ printf ' FROM `binary_packages` AS `subst_bp`'
+ mysql_join_binary_packages_install_target_providers 'subst_bp' 'subst_itp'
+ # ... in a repository ...
+ mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+ # ... more stable ...
+ printf ' JOIN `repository_stability_relations` AS `subst_rr`'
+ printf ' ON `subst_rr`.`more_stable`=`subst_r`.`stability`'
+ # ... than x's repository ...
+ printf ' WHERE `subst_rr`.`less_stable`=`repl_r`.`stability`'
+ printf ' AND NOT EXISTS ('
+ # ... and which is not replaced ...
+ printf 'SELECT * FROM `replaced_binary_packages_copy2`'
+ printf ' WHERE `replaced_binary_packages_copy2`.`id`=`subst_bp`.`id`'
+ printf ')'
+ # ... and provides the same
+ printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`'
+ # b) 2) ^
+ printf ');\n'
+ printf 'UNTIL row_count_saved=0 AND ROW_COUNT()=0\n'
+ printf 'END REPEAT;\n'
+ for table in 'moveable' 'replaced'; do
+ for copy in '' '2'; do
+ if [ "${table}" = 'moveable' ] && \
+ [ "${copy}" = '2' ]; then
+ continue
+ fi
+ printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy%s`;\n' \
+ "${table}" "${copy}"
+ done
+ done
+ printf 'END\n'
+ printf '//\n'
+ printf 'DELIMITER ;\n'
+
printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \
+ 'CREATE ROUTINE' 'buildmaster.*' \
'CREATE TEMPORARY TABLES' 'buildmaster.*' \
- 'SHOW VIEW' 'buildmaster.*' \
+ 'EXECUTE' 'buildmaster.*' \
+ 'RELOAD' '*.*' \
'SELECT' 'buildmaster.*' \
- 'UPDATE' 'buildmaster.*' \
- 'RELOAD' '*.*'
- printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \
+ 'SELECT' 'mysql.proc' \
'SHOW VIEW' 'buildmaster.*' \
- 'SELECT' 'buildmaster.*'
+ 'UPDATE' 'buildmaster.*'
+ printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \
+ 'CREATE TEMPORARY TABLES' 'buildmaster.*' \
+ 'SELECT' 'buildmaster.*' \
+ 'SHOW VIEW' 'buildmaster.*'
printf 'FLUSH PRIVILEGES;\n'
} | \
- mysql -u root -p buildmaster
+ mysql_run_query -u root -p
fi
+# shellcheck disable=SC2016
{
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `architectures` (`name`) VALUES '
printf '("%s"),' \
'any' 'i686' | \
sed 's|,$||'
printf ';\n'
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `fail_reasons` (`identifier`,`name`,`severity`) VALUES '
printf '(from_base64("%s"),"%s",%s),' \
"$(
@@ -276,7 +601,6 @@ fi
)" 'unknown' 100 | \
sed 's|,$||'
printf ';\n'
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `git_repositories` (`name`,`url`,`directory`,`head`) VALUES'
{
for repo in ${repo_names}; do
@@ -298,8 +622,8 @@ fi
base64 -w0
)" \
"$(
- base64_encode_each < \
- "${work_dir}/${repo}.revision"
+ git -C "${repo_path}" rev-parse HEAD | \
+ base64_encode_each
)" | \
sed 's|,$|),|'
done
@@ -314,9 +638,7 @@ fi
'extra:packages' \
'multilib:packages' \
'community:community'; do
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `upstream_repositories` (`name`,`git_repository`) SELECT\n'
- # shellcheck disable=SC2016
printf ' from_base64("%s"),`id` FROM `git_repositories` WHERE `name` = from_base64("%s");\n' \
"$(
printf '%s' "${repo%:*}" | \
@@ -328,7 +650,6 @@ fi
)"
done
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `build_slaves` (`name`,`ssh_key`,`operator`,`last_connection`) VALUES'
{
sed -n '
@@ -338,10 +659,13 @@ fi
' ~/.ssh/authorized_keys | \
while read -r name key; do
case "${name}" in
- 'nlopc'*|'rechenknecht'|'buildknecht'*)
+ 'nlopc'*|'rechenknecht')
operator='deep42thought'
;;
- 'eurobuild')
+ 'buildknecht'*)
+ operator='deep42thought/vollzornbrot'
+ ;;
+ 'eurobuild3')
operator='abaumann'
;;
*)
@@ -368,20 +692,69 @@ fi
} | \
sed 's|,;|;|'
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `repository_stabilities` (`name`,`bugtracker_category`) VALUES'
{
printf '\n ("%s",%s),' \
- 'stable' '"Packages:Stable"' \
- 'testing' '"Packages:Testing"' \
+ 'stable' '"Packages: Stable"' \
+ 'testing' '"Packages: Testing"' \
'staging' 'NULL' \
'standalone' 'NULL' \
- 'unbuilt' '"Packages:Build-list"' \
+ 'unbuilt' '"Packages: Build-list"' \
'forbidden' 'NULL'
printf ';\n'
} | \
sed 's|,;|;|'
+ {
+ printf 'INSERT IGNORE INTO `repository_stability_relations` (`more_stable`,`less_stable`)'
+ printf ' SELECT `ms`.`id`,`ls`.`id`'
+ printf ' FROM `repository_stabilities` AS `ms` JOIN `repository_stabilities` AS `ls`'
+ printf ' WHERE '
+ printf '(`ms`.`name`="%s" AND `ls`.`name`="%s") OR ' \
+ 'stable' 'stable' \
+ 'stable' 'testing' \
+ 'stable' 'staging' \
+ 'stable' 'standalone' \
+ 'stable' 'unbuilt' \
+ 'stable' 'forbidden' \
+ 'testing' 'testing' \
+ 'testing' 'staging' \
+ 'testing' 'standalone' \
+ 'testing' 'unbuilt' \
+ 'testing' 'forbidden' \
+ 'staging' 'staging' \
+ 'staging' 'standalone' \
+ 'staging' 'unbuilt' \
+ 'staging' 'forbidden' \
+ 'unbuilt' 'forbidden' \
+ 'standalone' 'standalone'
+ printf ';\n'
+ } | \
+ sed 's| OR ;|;|'
+
+ {
+ printf 'INSERT IGNORE INTO `repository_moves` (`from_repository`,`to_repository`,`upstream_package_repository`)'
+ printf ' SELECT `f`.`id`,`t`.`id`,`u`.`id`'
+ printf ' FROM'
+ printf ' `repositories` AS `%s` JOIN' \
+ 'f' 't'
+ printf ' `upstream_repositories` AS `u`'
+ printf ' WHERE '
+ printf '(`f`.`name`="%s" AND `t`.`name`="%s" AND `u`.`name`="%s") OR ' \
+ 'staging' 'testing' 'core' \
+ 'staging' 'testing' 'extra' \
+ 'staging' 'testing' 'multilib' \
+ 'community-staging' 'community-testing' 'community' \
+ 'community-staging' 'community-testing' 'multilib' \
+ 'testing' 'core' 'core' \
+ 'testing' 'extra' 'extra' \
+ 'testing' 'extra' 'multilib' \
+ 'community-testing' 'community' 'community' \
+ 'community-testing' 'community' 'multilib'
+ printf ';\n'
+ } | \
+ sed 's| OR ;|;|'
+
for repo in \
'core:stable:AQ==' \
'extra:stable:AQ==' \
@@ -393,9 +766,7 @@ fi
'community-staging:staging:AQ==' \
'build-list:unbuilt:AA==' \
'deletion-list:forbidden:AA=='; do
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `repositories` (`name`,`stability`,`is_on_master_mirror`) SELECT'
- # shellcheck disable=SC2016
printf ' from_base64("%s"),`id`,from_base64("%s") FROM `repository_stabilities` WHERE `name`=from_base64("%s");\n' \
"$(
printf '%s' "${repo}" | \
@@ -413,80 +784,18 @@ fi
)"
done
- # shellcheck disable=SC2016
printf 'INSERT IGNORE INTO `dependency_types` (`name`,`relevant_for_building`,`relevant_for_binary_packages`) VALUES'
{
printf '\n ("%s",%s,%s),' \
- 'build' '1' '0' \
+ 'make' '1' '0' \
+ 'check' '0' '0' \
+ 'link' '0' '1' \
'run' '1' '1'
printf ';\n'
} | \
sed 's|,;|;|'
} | \
- mysql buildmaster
-
-find "${work_dir}/package-states" \
- -maxdepth 1 \
- -mindepth 1 \
- \( \
- -name '*.done' -o \
- -name '*.testing' -o \
- -name '*.tested' \
- \) \
- -exec sed '
- 1!d
- s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@
- s@^\S*/@@
- s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 /
- s/ \([0-9]\+\) / \1.0 /
- s/ [0-9]\+\.\([0-9]\+\) / \1 /
- ' {} \; | \
- while read -r state_file sub_pkgrel arch package_file; do
- printf '%s ' "${state_file}"
- repository=$(
- grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \
- cut -d' ' -f1
- )
- if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then
- printf 'not found exactly once on the master mirror.\n'
- >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}"
- continue
- fi
- mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}"
- printf ' ok\n'
- done
-
-tr ' ' '.' < \
- "${work_dir}/build-list" | \
- while read -r state_file; do
- mysql_generate_package_metadata "${state_file}"
- done
-
-# shellcheck disable=SC2016
-find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \
- sed '
- s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3|
- ' | \
- while read -r state_file pkgbase git_revision mod_git_revision repository; do
- printf 'UPDATE `build_assignments`'
- printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
- 'package_sources' 'package_sources' 'build_assignments' 'package_source' \
- 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
- printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \
- "$(
- tr -d '\n' < \
- "${state_file}" | \
- base64 -w0
- )"
- printf ' WHERE'
- printf ' `package_sources`.`%s`=from_base64("%s") AND' \
- 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \
- 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \
- 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)"
- printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
- "$(printf '%s' "${repository}" | base64 -w0)"
- done | \
- ${mysql_command}
+ mysql_run_query
grep '^\('"$(
# shellcheck disable=SC2086
@@ -520,10 +829,9 @@ grep '^\('"$(
"${tmp_dir}/new-stable-packages"
if [ -s "${tmp_dir}/new-stable-packages" ]; then
+ # shellcheck disable=SC2016
{
- # shellcheck disable=SC2016
printf 'CREATE TEMPORARY TABLE `stable_packages` ('
- # shellcheck disable=SC2016
printf '`%s` %s,' \
'pkgname' 'VARCHAR(64)' \
'epoch' 'MEDIUMINT' \
@@ -533,9 +841,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
'architecture' 'VARCHAR(16)' \
'repository' 'VARCHAR(64)' \
'build_assignment' 'BIGINT NOT NULL AUTO_INCREMENT'
- # shellcheck disable=SC2016
printf 'PRIMARY KEY (`build_assignment`));\n'
- # shellcheck disable=SC2016
sed '
1~10 ! b not_start
s/^/INSERT IGNORE INTO `stable_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`) VALUES \n/
@@ -546,29 +852,25 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then
:end
s/,$/;/
' "${tmp_dir}/new-stable-packages"
- # shellcheck disable=SC2016
- printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`)'
+ printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)'
printf ' SELECT '
- # shellcheck disable=SC2016
printf '`stable_packages`.`%s`,' \
'pkgname' \
'epoch' \
'pkgver' \
'pkgrel' \
'sub_pkgrel'
- # shellcheck disable=SC2016
printf '`%s`.`id`,' \
'architectures' \
'repositories'
- # shellcheck disable=SC2016
- printf -- '-`build_assignment`,0,1 FROM `stable_packages`'
- # shellcheck disable=SC2016
+ printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`'
printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \
'repositories' 'repository' 'repositories' \
'architectures' 'architecture' 'architectures'
printf ';\n'
- # shellcheck disable=SC2016
printf 'DROP TABLE `stable_packages`;\n'
} | \
- ${mysql_command}
+ mysql_run_query
fi
+
+mysql_repair_binary_packages_without_build_assignment
diff --git a/bin/build-master-status b/bin/build-master-status
index 4d47976..c93a9aa 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -5,37 +5,31 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
+# TODO: replace by build-master-status-from-mysql
+
usage() {
>&2 echo ''
>&2 echo 'build-master-status: report about status of build master'
>&2 echo ''
>&2 echo 'possible options:'
- >&2 echo ' -w|--web:'
- >&2 echo ' Output to webserver instead of stdout.'
>&2 echo ' -h|--help:'
>&2 echo ' Show this help and exit.'
[ -z "$1" ] && exit 1 || exit "$1"
}
eval set -- "$(
- getopt -o hw \
+ getopt -o h \
--long help \
- --long web \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
-web=false
-
while true
do
case "$1" in
-h|--help)
usage 0
;;
- -w|--web)
- web=true
- ;;
--)
shift
break
@@ -61,538 +55,177 @@ fi
tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-stable=$(
- ls_master_mirror 'i686' | \
- grep -v 'testing$\|staging$\|-unstable$' | \
- while read -r dir; do
- ls_master_mirror "i686/${dir}"
- done | \
- grep -c '\.pkg\.tar\.xz$'
-)
-tasks=$(
- grep -c '^\S\+ \S\+ \S\+ \S\+$' \
- "${work_dir}/build-list"
-) || true
-pending_packages=$(
- grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \
- tr ' ' '.' | \
- while read -r package; do
- generate_package_metadata "${package}" 2>&1 > /dev/null
- cat "${work_dir}/package-infos/${package}.packages"
- done |
- wc -l
-)
-next_tasks=$(
- {
- cat "${work_dir}/build-list"
- find "${work_dir}/package-states" -maxdepth 1 \
- \( -name '*.broken' -o -name '*.blocked' \) \
- -printf '%f\n' | \
- sed '
- s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
- p
- '
- } | \
- sort | \
- uniq -u | \
- while read -r package git_revision mod_git_revision repository; do
- if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
- echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
- fi
- done | \
- wc -l
-)
-staging=$(
- find "${work_dir}/package-states" -name '*.done' \
- -exec cat '{}' \; | \
- sort -u | \
- wc -l
-)
-testing=$(
- find "${work_dir}/package-states" -name '*.testing' \
- -exec cat '{}' \; | \
- sort -u | \
- wc -l
-)
-tested=$(
- find "${work_dir}/package-states" -name '*.tested' \
- -exec cat '{}' \; | \
- sort -u | \
- wc -l
-)
{
- find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
- sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
- while read -r pkg rev mod_rev repo; do
- if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then
- echo "${pkg}"
- fi
- done
- {
- find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -exec cat '{}' \; | \
- sort -u
- find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
- sed 's|\(\.[^.]\+\)\{4\}||' | \
- sort -u
- } | \
- sort | \
- uniq -d
-} | \
- sort -u > \
- "${tmp_dir}/broken-packages-names"
-broken=$(
- wc -l < \
- "${tmp_dir}/broken-packages-names"
-)
-blocked=$(
- find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \
- wc -l
-)
-locked=$(
- find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \
- wc -l
-)
-loops=$(
- find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' | \
- wc -l
-)
-looped_packages=$(
- find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -exec cat '{}' \; | \
- sort -u | \
- wc -l
-)
-
-{
- printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \
- "${stable}" \
- "$((staging+testing+tested+pending_packages))"
- printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \
- "$((tasks-broken))" \
- "${tasks}" \
- "${pending_packages}" \
- "${next_tasks}"
- printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \
- "$((testing+tested))" \
- "${tested}" \
- "${staging}"
- printf 'There are %d broken package builds.\n' \
- "${broken}"
- if [ "${loops}" -ne 0 ]; then
- printf 'There are %d loops containing %d package builds.\n' \
- "${loops}" \
- "${looped_packages}"
- fi
- if [ $((broken+testing+tested+staging)) -ne 0 ]; then
- printf '%.1f%% of all packages are broken.\n' \
- "$(
- echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \
- bc
- )"
- fi
- if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then
- printf '%.1f%% of the planned work has been done.\n' \
- "$(
- echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \
- bc
- )"
- fi
-} > \
- "${tmp_dir}/build-master-status.html"
-
-if ${web}; then
- "${base_dir}/bin/calculate-dependent-packages"
- {
- printf '%s\n' \
- '<html>' \
- '<head>' \
- '<title>Status of archlinux32 build master</title>' \
- '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
- '</head>' \
- '<body>'
- sed 's|$|<br>|' "${tmp_dir}/build-master-status.html"
- printf '%s\n' \
- '<br>' \
- 'currently building packages:<br>' \
- '<table>'
- printf '<tr>'
- printf '<th>%s</th>' \
- 'since (UTC)' \
- 'pkgname' \
- 'git revision' \
- 'modification git revision' \
- 'package repository' \
- 'build slave'
- printf '</tr>'
- find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \
- -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \
- -execdir sed '
- :a
- $!{
- N
- s/\n/, /
- ba
- }
- ' '{}' \; | \
- sort -k1n,1 | \
- sed '
- s|^\S\+ ||
- s|\.locked | |
- s|\.\([^.]\+\)$| \1|
- s|\.\([^.]\+\)$| \1|
- s|\.\([^.]\+\)$| \1|
- ' | \
- while read -r date time pkg rev mod_rev repo slaves; do
- printf '<tr>'
- printf '<td>%s</td>' \
- "${date} ${time}" \
- "${pkg}" \
- "<p style=\"font-size:8px\">${rev}</p>" \
- "<p style=\"font-size:8px\">$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")</p>" \
- "${repo}" \
- "${slaves}"
- printf '</tr>\n'
- done
- printf '%s\n' \
- '</table>' \
- '</body>' \
- '</html>'
- } | \
- sponge "${tmp_dir}/build-master-status.html"
- end=$(($(date +%s)-7*24*60*60))
- {
- [ -f "${webserver_directory}/statistics" ] && \
- cat "${webserver_directory}/statistics"
- printf '%s ' \
- "$(date +%s)" \
- "${stable}" \
- "${tasks}" \
- "${pending_packages}" \
- "${staging}" \
- "${testing}" \
- "${broken}" \
- "${loops}" \
- "${looped_packages}" \
- "${locked}" \
- "${blocked}" \
- "${next_tasks}" \
- "${tested}" | \
- sed 's| $|\n|'
- echo "${end}"
- } | \
- sort -k1nr,1 | \
- sed -n "
- /^${end}\$/q
+ printf '%s\n' \
+ '<html>' \
+ '<head>' \
+ '<title>Todos in the build scripts</title>' \
+ '</head>' \
+ '<body>'
+ find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
+ -exec grep -nHF '' '{}' \; | \
+ awk '
+ { print $0 }
+ /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i}
+ ' | \
+ sed -n '
+ s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/
+ T
+ N
+ s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/
+ :a
+ N
+ s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/
+ ta
+ s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
p
- " | \
- tac > \
- "${tmp_dir}/statistics"
-
- find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \
- \( \
- -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \
- -printf '%f build()\n' \
- \) -o \
- \( \
- -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \
- -printf '%f check()\n' \
- \) -o \
- \( \
- -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \
- -printf '%f prepare()\n' \
- \) -o \
- \( \
- -exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \
- -printf '%f package()\n' \
- \) -o \
- \( \
- -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \
- -printf '%f source\n' \
- \) -o \
- \( \
- -exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \
- -printf '%f dependencies\n' \
- \) -o \
- \( \
- -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \
- -printf '%f package-cache\n' \
- \) -o \
- \( \
- -exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \
- -printf '%f run-as-root\n' \
- \) -o \
- -printf '%f unknown\n' \
- \) | \
- sed '
- s|\(\.[^.]\+\)\{3\} | |
' | \
- sort -u | \
+ tee "${tmp_dir}/todos" | \
sed '
:a
- $!N
- s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/
- ta
- P
- D
+ N
+ /\n$/!ba
+ s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n|
' | \
- sort -k1,1 > \
- "${tmp_dir}/broken-packages.reason"
-
- {
- printf '%s\n' \
- '<html>' \
- '<head>' \
- '<title>List of broken package builds</title>' \
- '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
- '</head>' \
- '<body>' \
- '<a href="build-logs/">build logs</a><br>' \
- '<table>' \
- '<tr>'
- printf '<th>%s</th>' \
- 'package' \
- 'git revision' \
- 'modification git revision' \
- 'package repository' \
- 'compilations' \
- 'dependent' \
- 'build error' \
- 'blocked'
- printf '</tr>\n'
- find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
- sed 's|\.broken$||' | \
- sort -k1,1 | \
- join -j 1 - "${tmp_dir}/broken-packages.reason" | \
- sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \
- while read -r sf pkg rev mod_rev repo build_error; do
- if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then
- printf '1 '
- else
- printf '0 '
- fi
- printf '%s ' \
- "${pkg}" \
- "${rev}" \
- "${mod_rev}" \
- "${repo}" \
- "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \
- "$(
- # shellcheck disable=SC2010
- ls -t "${webserver_directory}/build-logs/error" | \
- grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$'
- )" \
- "$(
- {
- grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \
- echo 'x &nbsp;'
- } | \
- cut -d' ' -f2
- )" \
- "${build_error}"
- if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then
- sed '
- s|\s\(wait for \)|\n\1|g
- ' "${work_dir}/package-states/${sf}.blocked" | \
- while read -r blocked_reason; do
- if echo "${blocked_reason}" | \
- grep -q '^wait for '; then
- printf 'wait for '
- echo "${blocked_reason}" | \
- sed '
- s|^wait for ||
- s@\( and \| or \)@\n\1\n@
- ' | \
- while read -r reason; do
- if [ "FS#${reason#FS#}" = "${reason}" ]; then
- printf '<a href="https://bugs.archlinux.org/task/%s">%s</a>' \
- "${reason#FS#}" \
- "${reason}"
- elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then
- printf '<a href="https://bugs.archlinux32.org/index.php?do=details&task_id=%s">%s</a>' \
- "${reason#FS32#}" \
- "${reason}"
- elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then
- printf '<a href="graphs/%s.png">%s</a>' \
- "${reason}" \
- "${reason}"
- elif [ "${reason% *}" != "${reason}" ]; then
- printf '%s' \
- "${reason}"
- else
- printf '<font color="red">%s</font>' \
- "${reason}"
- fi
- if read -r operator; then
- printf ' %s ' "${operator}"
- fi
- done
- else
- echo "${blocked_reason}"
- fi
- done | \
- tr '\n' ' '
- else
- printf '&nbsp;'
- fi
- printf '\n'
- done | \
- sort -k6n,6 | \
- while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do
- if [ "${buildable}" -eq 0 ]; then
- left='('
- right=')'
- else
- unset left
- unset right
- fi
- printf '<tr>'
- mod_rev=$(
- modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
- )
- build_error=$(
- echo "${build_error}" | \
- sed 's|,|, |g'
- )
- printf '<td>%s</td>' \
- '<a href="graphs/'"${pkg}"'.png">'"${left}${pkg}${right}"'</a>' \
- "<p style=\"font-size:8px\">${rev}</p>" \
- "<p style=\"font-size:8px\">${mod_rev}</p>" \
- "${repo}" \
- '<a href="build-logs/error/'"${log_file}"'">'"${count}"'</a>' \
- "${dependent}" \
- "${build_error}" \
- "${reason}"
- printf '</tr>\n'
- done
- printf '%s\n' \
- '</table>' \
- '</body>' \
- '</html>'
- } > \
- "${tmp_dir}/broken-packages.html"
-
- rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason"
-
+ sed '
+ s|$|<br>|
+ '
+ printf '%s\n' \
+ '</body>' \
+ '</html>'
+} > \
+ "${tmp_dir}/todos.html"
+
+if [ -s "${tmp_dir}/todos" ]; then
+ sed '
+ :a
+ N
+ /\n$/!ba
+ s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 |
+ s/\n$//
+ s/\n/\\n/g
+ ' -i "${tmp_dir}/todos"
+ while read -r file line desc; do
+ printf '%s %s %s\n' \
+ "$(printf '%s' "${file}" | base64 -w0)" \
+ "$(printf '%s' "${line}" | base64 -w0)" \
+ "$(printf '%s' "${desc}" | base64 -w0)"
+ done < \
+ "${tmp_dir}/todos" | \
+ sponge "${tmp_dir}/todos"
+ # update todos
+ # shellcheck disable=SC2016
+ while read -r file line desc; do
+ printf 'UPDATE IGNORE `todos`'
+ printf ' SET `todos`.`line`=from_base64("%s")' \
+ "${line}"
+ printf ' WHERE `todos`.`file`=from_base64("%s")' \
+ "${file}"
+ printf ' AND `todos`.`description`=from_base64("%s");\n' \
+ "${desc}"
+
+ printf 'UPDATE IGNORE `todos`'
+ printf ' SET `todos`.`description`=from_base64("%s")' \
+ "${desc}"
+ printf ' WHERE `todos`.`file`=from_base64("%s")' \
+ "${file}"
+ printf ' AND `todos`.`line`=from_base64("%s");\n' \
+ "${line}"
+ done < \
+ "${tmp_dir}/todos" | \
+ mysql_run_query
+ # insert unfound todos
+ # shellcheck disable=SC2016
{
- printf '%s\n' \
- '<html>' \
- '<head>' \
- '<title>Todos in the build scripts</title>' \
- '</head>' \
- '<body>'
- find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
- -exec grep -nHF '' '{}' \; | \
- awk '
- { print $0 }
- /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i}
- ' | \
- sed -n '
- s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/
- T
- N
- s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/
- :a
- N
- s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/
- ta
- s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
- p
- ' | \
+ printf 'SHOW CREATE TABLE `todos`' | \
+ mysql_run_query | \
sed '
- :a
- N
- /\n$/!ba
- s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n|
- ' | \
- sed '
- s|$|<br>|
+ 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` /
'
- printf '%s\n' \
- '</body>' \
- '</html>'
- } > \
- "${tmp_dir}/todos.html"
-
- {
- printf '%s\n' \
- '<html>' \
- '<head>' \
- '<title>Blacklisted packages</title>' \
- '<link rel="stylesheet" type="text/css" href="/static/style.css">' \
- '</head>' \
- '<body>' \
- '<table>'
- printf '<tr>'
- printf '<th>%s</th>' \
- 'package' \
- 'reason'
- printf '</tr>\n'
- git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
- tar -Ox | \
+ printf ';\n'
+ printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES '
+ while read -r file line desc; do
+ printf '('
+ printf 'from_base64("%s"),' \
+ "${file}" \
+ "${line}" \
+ "${desc}" | \
+ sed 's/,$/),/'
+ done < \
+ "${tmp_dir}/todos" | \
sed '
- s@FS#\([0-9]\+\)@<a href="https://bugs.archlinux.org/task/\1">\0</a>@
- s@FS32#\([0-9]\+\)@<a href="https://bugs.archlinux32.org/index.php?do=details\&task_id=\1">\0</a>@
- /.#/!s/$/#/
- s|\(.\)#|\1</td><td>|
- /^\s*#/{
- s/^\s*#\s*//
- s|\s*\(</td><td>\)|</font></s>\1|
- s/^/<s><font color="#808080">/
- }
- s|^|<tr><td>|
- s|$|</td></tr>|
+ s/,$//
'
- printf '%s\n' \
- '</table>' \
- '</body>' \
- '</html>'
- } > \
- "${tmp_dir}/blacklist.html"
-
- {
- printf '%s\n' \
- '<html>' \
- '<head>' \
- '<title>log of ssh connections from build slaves</title>' \
- '</head>' \
- '<body>' \
- '<table>'
- printf '<tr>'
- printf '<th>%s</th>' \
- 'time' \
- 'build slave' \
- 'command' \
- 'arguments'
- printf '</tr>\n'
- if [ -r "${work_dir}/ssh-log" ]; then
- tac "${work_dir}/ssh-log" | \
- while read -r date time slave command arguments; do
- printf '<tr>'
- printf '<td>%s</td>' \
- "${date} ${time}" \
- "${slave}" \
- "${command}" \
- "${arguments}"
- printf '</tr>\n'
- done
- fi
- printf '%s\n' \
- '</table>' \
- '</body>' \
- '</html>'
- } > \
- "${tmp_dir}/ssh-log.html"
+ printf ';\n'
+ printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) '
+ printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` '
+ printf 'FROM `td` '
+ printf 'WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `todos`'
+ printf ' AND `td`.`%s`=`todos`.`%s`' \
+ 'file' 'file' \
+ 'line' 'line' \
+ 'description' 'description' | \
+ sed 's/^ AND / WHERE /'
+ printf ');\n'
+
+ printf 'DELETE FROM `todos` WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `td`'
+ printf ' AND `td`.`%s`=`todos`.`%s`' \
+ 'file' 'file' \
+ 'line' 'line' \
+ 'description' 'description' | \
+ sed 's/^ AND / WHERE /'
+ printf ');'
+ printf 'DROP TABLE `td`;\n'
+ printf 'DELETE FROM `todo_links` WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `todos` '
+ printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`'
+ printf ') OR NOT EXISTS ('
+ printf 'SELECT * FROM `todos` '
+ printf 'WHERE `todos`.`id`=`todo_links`.`dependent`'
+ printf ');\n'
+ } | \
+ mysql_run_query
+ rm -f "${tmp_dir}/todos"
+fi
- find "${tmp_dir}" -maxdepth 1 -type f | \
- while read -r file; do
- cat "${file}" > \
- "${webserver_directory}/${file##*/}"
- done
+{
+ printf '%s\n' \
+ '<html>' \
+ '<head>' \
+ '<title>log of ssh connections from build slaves</title>' \
+ '</head>' \
+ '<body>' \
+ '<table>'
+ printf '<tr>'
+ printf '<th>%s</th>' \
+ 'time' \
+ 'build slave' \
+ 'command' \
+ 'arguments'
+ printf '</tr>\n'
+ if [ -r "${work_dir}/ssh-log" ]; then
+ tac "${work_dir}/ssh-log" | \
+ while read -r date time slave command arguments; do
+ printf '<tr>'
+ printf '<td>%s</td>' \
+ "${date} ${time}" \
+ "${slave}" \
+ "${command}" \
+ "${arguments}"
+ printf '</tr>\n'
+ done
+ fi
+ printf '%s\n' \
+ '</table>' \
+ '</body>' \
+ '</html>'
+} > \
+ "${tmp_dir}/ssh-log.html"
-else
- cat "${tmp_dir}/build-master-status.html"
-fi
+find "${tmp_dir}" -maxdepth 1 -type f | \
+ while read -r file; do
+ cat "${file}" > \
+ "${webserver_directory}/${file##*/}"
+ done
diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index a0c2b4f..962b12e 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -5,30 +5,23 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
+# TODO: add all (necessary) features from build-master-status
+
+tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir)
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+# do not block if locked
+exec 9> "${sanity_check_lock_file}"
+if ! flock -n 9; then
+ >&2 echo 'Mysql-Sanity check skipped, cannot acquire lock.'
+ exit
+fi
+
+# shellcheck disable=SC2119
+mysql_cleanup
+
{
mysql_sanity_check || true
- {
- # shellcheck disable=SC2016
- {
- printf 'SELECT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`'
- printf ' FROM `package_sources`'
- printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \
- 'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \
- 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \
- 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \
- 'repositories' 'binary_packages' 'repository' 'repositories'
- printf ' WHERE `repositories`.`name`="build-list"'
- } | \
- ${mysql_command} --batch | \
- sed '
- 1d
- y/\t/ /
- s/^/+ /
- '
- sed 's/^/- /' "${work_dir}/build-list"
- } | \
- sort -k2 -k1,1 | \
- uniq -uf1
} | \
sed '
s,^-.*$,<font color="#FF0000">\0</font>,
@@ -38,3 +31,118 @@
$ a </body></html>
' | \
sponge "${webserver_directory}/mysql-sanity.html"
+
+if [ -s "${webserver_directory}/mysql-sanity.html" ] && \
+ [ ! -s "${work_dir}/build-master-sanity" ]; then
+ printf 'girls, my database is dirty again ...\n' | \
+ irc_say
+ echo 'build master is insane' > \
+ "${work_dir}/build-master-sanity"
+fi
+
+if [ ! -s "${work_dir}/build-master-sanity" ]; then
+ # shellcheck disable=SC2016
+ {
+ printf 'INSERT IGNORE INTO `statistics` ('
+ printf '`%s`,' \
+ 'date' \
+ 'stable_packages_count' \
+ 'pending_tasks_count' \
+ 'pending_packages_count' \
+ 'staging_packages_count' \
+ 'testing_packages_count' \
+ 'tested_packages_count' \
+ 'broken_tasks_count' \
+ 'dependency_loops_count' \
+ 'dependency_looped_tasks_count' \
+ 'locked_tasks_count' \
+ 'blocked_tasks_count' \
+ 'next_tasks_count' | \
+ sed 's/,$//'
+ printf ') VALUES ('
+ # date
+ printf 'NOW(),'
+ # stable_packages_count
+ printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+ printf ' `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' WHERE `repository_stabilities`.`name`="stable"),'
+ # pending_tasks_count
+ printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+ printf ' `build_assignments`'
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"),'
+ # pending_packages_count
+ printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+ printf ' `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"),'
+ # staging_packages_count
+ printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+ printf ' `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' WHERE `repository_stabilities`.`name`="staging"),'
+ # testing_packages_count
+ printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+ printf ' `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' WHERE `repository_stabilities`.`name`="testing"'
+ printf ' AND NOT `binary_packages`.`is_tested`),'
+ # tested_packages_count
+ printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM'
+ printf ' `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' WHERE `repository_stabilities`.`name`="testing"'
+ printf ' AND `binary_packages`.`is_tested`),'
+ # broken_tasks_count
+ printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+ printf ' `build_assignments`'
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND `build_assignments`.`is_broken`),'
+ # dependency_loops_count
+ printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM'
+ printf ' `build_dependency_loops`),'
+ # dependency_looped_tasks_count
+ printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM'
+ printf ' `build_dependency_loops`),'
+ # locked_tasks_count
+ printf '(SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM'
+ printf ' `build_slaves`'
+ mysql_join_build_slaves_build_assignments
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"),'
+ # blocked_tasks_count
+ printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+ printf ' `build_assignments`'
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND `build_assignments`.`is_blocked` IS NOT NULL),'
+ # next_tasks_count
+ printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM'
+ printf ' `build_assignments`'
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND NOT EXISTS ('
+ printf 'SELECT * FROM `dependencies`'
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ mysql_join_dependencies_install_target_providers
+ mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+ mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+ printf ' WHERE `prov_r`.`name`="build-list"'
+ printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`'
+ printf '))'
+ printf ');\n'
+ } | \
+ mysql_run_query
+fi
diff --git a/bin/build-packages b/bin/build-packages
index 81afa7f..02334e9 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -7,8 +7,7 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
-# TODO: report back in regular intervals, so the build master can
-# abort if the package is outdated and/or someone else finished it.
+# TODO: build other 'architectures', too (pentium4, i486)
# shellcheck disable=SC2016
usage() {
@@ -220,6 +219,11 @@ while [ "${count}" -ne 0 ]; do
if [ "${mod_git_revision}" = 'work-tree' ]; then
mod_git_revision=$(
# we can't just create an empty index-file with mktemp, because git doesn't like it
+ find . \
+ -mindepth 1 \
+ -maxdepth 1 \
+ -name 'tmp.build-packages.git.*' \
+ -exec rm -rf --one-file-system {} \;
tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir)
trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT
export GIT_INDEX_FILE="${tmp_subdir}/index.new"
@@ -251,8 +255,18 @@ while [ "${count}" -ne 0 ]; do
fi
cd "${base_dir}"
recursively_umount_and_rm "${tmp_dir}"
+ flock -u 9 || true
exit "${err}"
}
+ find "${work_dir}" \
+ -mindepth 1 \
+ -maxdepth 1 \
+ -name 'tmp.build-packages.??????' \
+ -printf '%p\n' | \
+ while read -r old_tmp_dir; do
+ find "${old_tmp_dir}" -xdev -exec chmod 777 {} \;
+ rm -rf --one-file-system "${old_tmp_dir}"
+ done
tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
trap bail_out EXIT
@@ -295,7 +309,19 @@ while [ "${count}" -ne 0 ]; do
# we can't improve anything
continue
fi
- tar -xz --overwrite -f "${source_name}" --exclude PKGBUILD --strip-components=1 || true
+ # shellcheck disable=SC2046
+ tar -xz --overwrite \
+ -f "${source_name}" \
+ --exclude PKGBUILD \
+ $(
+ if [ -n "${PKGBUILD_mod}" ]; then
+ git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \
+ tar -t | \
+ sed 's/^/--exclude /'
+ fi
+ ) \
+ --strip-components=1 \
+ || true
fi
if echo "${straw}" | \
@@ -321,19 +347,39 @@ while [ "${count}" -ne 0 ]; do
if echo "${straw}" | \
grep -qF ':clean_chroot:'; then
- parameters='-c'
+ outerParameters='-c'
else
- parameters=''
+ outerParameters=''
fi
if echo "${straw}" | \
grep -qF ':without_check:'; then
- parameters="${parameters} -- -- --nocheck"
+ innerParameters='--nocheck'
+ else
+ innerParameters=''
+ fi
+
+ if echo "${straw}" | \
+ grep -qF ':with_/dev/fuse:'; then
+ middleParameters='-d /dev/fuse'
+ else
+ middleParameters=''
fi
find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \
rm {} \;
+ echo 'building' > "${tmp_dir}/.ping-build-master"
+ if [ -z "${forced_package}" ]; then
+ # we get a lock on "${work_dir}/ping-build-master.lock",
+ # if we release that lock, ping-to-master should stop _immediately_
+ exec 9> "${work_dir}/ping-build-master.lock"
+ if ! flock -n 9; then
+ >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.'
+ exit 2
+ fi
+ "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" &
+ fi
>&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) ...' \
"$(date +'%Y-%m-%d %T')" \
"${package}" \
@@ -344,7 +390,7 @@ while [ "${count}" -ne 0 ]; do
# by piping the log, we don't see anything in the terminal,
# but all ways to duplicate the logs seem pretty elaborate
# shellcheck disable=SC2024,SC2086
- if sudo "${build_command}" ${parameters} > \
+ if sudo "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \
"$(
date -u --iso-8601=seconds | \
cut -d+ -f1
@@ -353,6 +399,7 @@ while [ "${count}" -ne 0 ]; do
>&2 printf ' ok.\n'
tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX")
find . -maxdepth 1 -type f -name '*-debug-*.pkg.tar.xz*' -delete
+ echo 'post-build' > "${tmp_dir}/.ping-build-master"
>&2 printf 'signing package(s)\n'
find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
-execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \
@@ -390,12 +437,50 @@ while [ "${count}" -ne 0 ]; do
>&2 printf ' failed. Next ...\n'
done
done
+ >&2 printf 'searching for provided libraries\n'
+ find "${tar_content_dir}" -maxdepth 1 \
+ -name '*.pkg.tar.xz' \
+ -printf '%p\n' | \
+ while read -r pkgfile; do
+ pacman -Qqlp "${pkgfile}" | \
+ sed -n '
+ s,^.*/,,
+ /\.so\(\..\+\)\?$/p
+ ' > \
+ "${pkgfile}.so.provides"
+ done
+ >&2 printf 'searching for required libraries\n'
+ package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX")
+ find "${tar_content_dir}" -maxdepth 1 \
+ -name '*.pkg.tar.xz' | \
+ while read -r pkgfile; do
+ if printf '%s\n' "${pkgfile}" | \
+ grep -vq -- '-any\.pkg\.tar\.xz$'; then
+ # we do not check "any" packages for linked libraries
+ # (why do they have them in the first place?)
+ mkdir "${package_content_dir}/${pkgfile##*/}"
+ tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
+ find "${package_content_dir}/${pkgfile##*/}" \
+ -name 'opt' -prune , \
+ -exec objdump -x '{}' \; 2>/dev/null | \
+ grep -w 'NEEDED' | \
+ awk '{print $2}' | \
+ sed '
+ /\.c32$/d
+ s,^.*/,,
+ '
+ find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 '{}' \;
+ rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
+ fi | \
+ sort -u > \
+ "${pkgfile}.so.needs"
+ done
+ >&2 printf 'running namcap ...'
if [ "${repository}" = 'multilib' ]; then
x86_64_build_command='multilib-build'
else
x86_64_build_command='extra-x86_64-build'
fi
- >&2 printf 'running namcap ...'
# this is a little hack: makepkg receives '--version', but namcap is run nevertheless
# (and it only works with devtools32, because they are running namcap on *.pkg.tar.xz in the base directory, too)
sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \
@@ -426,14 +511,31 @@ while [ "${count}" -ne 0 ]; do
fi
done
>&2 printf ' ok.\n'
+ echo 'uploading' > "${tmp_dir}/.ping-build-master"
if ${upload_to_build_master}; then
- find "${tar_content_dir}/" -maxdepth 1 -name '*.pkg.tar.xz-namcap.log' -execdir gzip '{}' \;
+ find "${tar_content_dir}/" -maxdepth 1 \
+ \( \
+ -name '*.pkg.tar.xz-namcap.log' -o \
+ -name '*.pkg.tar.xz.so.needs' -o \
+ -name '*.pkg.tar.xz.so.provides' \
+ \) \
+ -execdir gzip '{}' \;
else
- find "${tar_content_dir}/" -maxdepth 1 -name '*.pkg.tar.xz-namcap.log' -execdir grep -HF '' '{}' \;
+ find "${tar_content_dir}/" -maxdepth 1 \
+ -name '*.pkg.tar.xz-namcap.log' \
+ -execdir grep -HF '' '{}' \;
fi
# shellcheck disable=SC2046
tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
- find "${tar_content_dir}/" -maxdepth 1 \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' -o -name '*.pkg.tar.xz-namcap.log.gz' \) -printf '%f\n'
+ find "${tar_content_dir}/" -maxdepth 1 \
+ \( \
+ -name '*.pkg.tar.xz' -o \
+ -name '*.pkg.tar.xz.sig' -o \
+ -name '*.pkg.tar.xz-namcap.log.gz' -o \
+ -name '*.pkg.tar.xz.so.needs.gz' -o \
+ -name '*.pkg.tar.xz.so.provides.gz' \
+ \) \
+ -printf '%f\n'
)
while ${upload_to_build_master}; do
err=0
@@ -474,6 +576,7 @@ while [ "${count}" -ne 0 ]; do
success=true
break
fi
+ echo 'failure' > "${tmp_dir}/.ping-build-master"
>&2 printf ' failed.\n'
done
@@ -528,6 +631,7 @@ while [ "${count}" -ne 0 ]; do
# clean up tmp_dir
cd "${base_dir}"
recursively_umount_and_rm "${tmp_dir}"
+ flock -u 9 || true
trap - EXIT
continue
diff --git a/bin/calculate-dependent-packages b/bin/calculate-dependent-packages
deleted file mode 100755
index bfb87a1..0000000
--- a/bin/calculate-dependent-packages
+++ /dev/null
@@ -1,184 +0,0 @@
-#!/bin/sh
-
-# shellcheck source=conf/default.conf
-. "${0%/*}/../conf/default.conf"
-
-if [ -s "${work_dir}/build-master-sanity" ]; then
- >&2 echo 'Build master is not sane.'
- exit
-fi
-
-tmp_dir=$(mktemp -d 'tmp.calculate-dependent-packages.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-sort -k1,1 -u "${work_dir}/build-list" > \
- "${tmp_dir}/build-list"
-
-while read -r pkg rev mod_rev repo; do
- generate_package_metadata "${pkg}" "${rev}" "${mod_rev}" "${repo}"
-done < \
- "${tmp_dir}/build-list"
-
-mkdir "${tmp_dir}/loops"
-find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' | \
- while read -r loop; do
- sort -u "${loop}" | \
- join -j 1 - "${tmp_dir}/build-list" | \
- tr ' ' '.' > \
- "${tmp_dir}/loops/${loop##*/}"
- done
-
-touch "${tmp_dir}/dependent-count"
-
-tr ' ' '.' < \
- "${tmp_dir}/build-list" | \
- sponge "${tmp_dir}/build-list"
-
-sums=''
-
-while [ -s "${tmp_dir}/build-list" ] && [ "${sums}" != "$(sha512sum "${tmp_dir}/dependent-count")" ]; do
-
- sums=$(
- sha512sum "${tmp_dir}/dependent-count"
- )
-
- {
- sed 's|^|? |' "${tmp_dir}/build-list"
- sed 'p' "${tmp_dir}/dependent-count"
- } | \
- sort -k2,2 | \
- uniq -uf1 | \
- cut -d' ' -f2 | \
- sponge "${tmp_dir}/build-list"
-
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.build-depends|
- ' "${tmp_dir}/build-list" | \
- xargs -r grep -hvx 'base\|base-devel' | \
- sort -u > \
- "${tmp_dir}/build-list.build-depends"
-
- sed '
- s|^\S\+ |'"${work_dir}"'/package-infos/|
- s|$|.build-depends|
- ' "${tmp_dir}/dependent-count" | \
- xargs -r grep -Hvx 'base\|base-devel' | \
- sed '
- s|^.*/||
- s|\.build-depends:| |
- ' | \
- sort -k2,2 > \
- "${tmp_dir}/dependent-count.build-depends"
-
- rm -f "${tmp_dir}/dependent-count.new"
- touch "${tmp_dir}/dependent-count.new"
-
- while read -r sf; do
- if [ -n "$(
- {
- sort -u "${work_dir}/package-infos/${sf}.builds"
- cat "${tmp_dir}/build-list.build-depends"
- } | \
- sort | \
- uniq -d
- )" ]; then
- continue
- fi
-
- count="/$(
- sort -u "${work_dir}/package-infos/${sf}.builds" | \
- join -1 1 -2 2 -o 2.1 - "${tmp_dir}/dependent-count.build-depends" | \
- sort -u | \
- join -1 1 -2 2 -o 2.1,2.2 - "${tmp_dir}/dependent-count" | \
- tr '/ ' '\n' | \
- grep -vxF '' | \
- sort -u | \
- tr '\n' '/'
- )"
- printf '%s %s\n' \
- "${count}" \
- "${sf}" >> \
- "${tmp_dir}/dependent-count.new"
- find "${tmp_dir}/loops" -type f -maxdepth 1 \
- -exec grep -qxF "${sf}" {} \; \
- -exec rm {} \;
- done < \
- "${tmp_dir}/build-list"
-
- find "${tmp_dir}/loops" -maxdepth 1 | \
- while read -r loop; do
- if [ ! -f "${loop}" ]; then
- continue
- fi
- if [ -n "$(
- {
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.builds|
- ' "${loop}" | \
- xargs -r cat | \
- sort -u
- {
- sed 'p' "${loop}"
- cat "${tmp_dir}/build-list"
- } | \
- sort | \
- uniq -u | \
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.build-depends|
- ' | \
- xargs -r grep -hvx 'base\|base-devel' | \
- sort -u
- } | \
- sort | \
- uniq -d
- )" ]; then
- continue
- fi
-
- count="/$(
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.builds|
- ' "${loop}" | \
- xargs -r cat | \
- sort -u | \
- join -1 1 -2 2 -o 2.1 - "${tmp_dir}/dependent-count.build-depends" | \
- sort -u | \
- join -1 1 -2 2 -o 2.1,2.2 - "${tmp_dir}/dependent-count" | \
- tr '/ ' '\n' | \
- grep -vxF '' | \
- sort -u | \
- tr '\n' '/'
- )"
- while read -r sf; do
- printf '%s %s\n' "${count}" "${sf}" >> \
- "${tmp_dir}/dependent-count.new"
- find "${tmp_dir}/loops" -maxdepth 1 -type f -not -name "${loop##*/}" \
- -exec grep -qxF "${sf}" {} \; \
- -exec rm {} \;
- done < \
- "${loop}"
-
- rm "${loop}"
- done
-
- cat "${tmp_dir}/dependent-count" "${tmp_dir}/dependent-count.new" | \
- sort -k2,2 | \
- sponge "${tmp_dir}/dependent-count"
-
-done
-
-while read -r count pkg; do
- count=$(
- echo "${count}" | \
- tr '/' '\n' | \
- grep -cvxF ''
- ) || true
- printf '%s %s\n' "${pkg}" "${count}"
-done < \
- "${tmp_dir}/dependent-count" | \
- sponge "${work_dir}/dependent-count"
diff --git a/bin/check-bugtracker b/bin/check-bugtracker
index 46e78db..5bd05b4 100755
--- a/bin/check-bugtracker
+++ b/bin/check-bugtracker
@@ -1,33 +1,64 @@
#!/bin/sh
# check the bug tracker for packages in testing and community-testing
-# with issues and mark these packages as "testing" if they are currently
-# marked as "tested"
+# with issues and mark these packages as "has_issues" if they are faulty
+# and vice versa
# shellcheck disable=SC2039
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
-tmp_dir=$(mktemp -d 'tmp.check-bugtracker.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+bug_list=$(
+ curl -LSs 'https://bugs.archlinux32.org/index.php?export_list=Export%20Tasklist' | \
+ sed -n '
+ 1d
+ s/^[^,]\+,"//
+ T
+ s/^\([^"]\+\)"\(,[^,]\+\)\{2\},"\([^"]\+\)".*$/"\1" "\3"/
+ T
+ p
+ '
+)
-receive_buglist 'Testing' | \
- tr ' ,;' '\n' | \
- sed -n '
- s/^\[//
- T
- s/]$//
- T
- p
- ' | \
- sort -u > \
- "${tmp_dir}/faulty-packages"
-
-find "${work_dir}/package-states" -name '*.tested' -printf '%f\n' | \
- sed '
- s|\.[^.]\+$||
- s|^\(.*\)\(\.[^.]\+\)\{3\}$|\1 \0|
- ' | \
- sort -k1,1 | \
- join -1 1 -2 1 -o 1.2 - "${tmp_dir}/faulty-packages" | \
- "${base_dir}/bin/modify-package-state" -n --faulty /dev/stdin
+# shellcheck disable=SC2016
+{
+ printf 'SELECT `repository_stabilities`.`id`,`repository_stabilities`.`bugtracker_category`'
+ printf ' FROM `repository_stabilities`'
+ printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL'
+} | \
+ mysql_run_query | \
+ while read -r stability_id category; do
+ for has_issues in '1:' '0:NOT '; do
+ printf 'UPDATE `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' SET `has_issues`=%s' \
+ "${has_issues%:*}"
+ printf ' WHERE `repositories`.`stability`=%s' \
+ "${stability_id}"
+ printf ' AND `binary_packages`.`pkgname` %sIN (' \
+ "${has_issues#*:}"
+ printf '%s\n' "${bug_list}" | \
+ sed -n '
+ s/^"'"$(str_to_regex "${category}")"'" //
+ T
+ :a
+ /\[.*]/ {
+ s/^[^[]*\[//
+ T
+ h
+ s/].*$//
+ p
+ x
+ ba
+ }
+ ' | \
+ base64_encode_each | \
+ sed '
+ s/^/from_base64("/
+ s/$/")/
+ ' | \
+ tr '\n' ','
+ printf '"");\n'
+ done
+ done | \
+ mysql_run_query
diff --git a/bin/check-opcodes b/bin/check-opcodes
index c7ab4a9..bd96cc6 100755
--- a/bin/check-opcodes
+++ b/bin/check-opcodes
@@ -101,7 +101,7 @@ log "Checking for architecture: $ARCH ($OPCODE_ARGS)"
bsdtar --no-fflags -x -C $tmp_dir -f $PACKAGE
# shellcheck disable=SC2044
-for absfile in $(find $tmp_dir -name '*.so*' -type f); do
+for absfile in $(find $tmp_dir -regextype grep -regex '.*\.so\(\.[0-9.]\+\)\?' -type f); do
file=$(basename $absfile)
log "Checking shared library: $file"
readelf -a $absfile > $tmp_dir/$file.elf
diff --git a/bin/cleanup b/bin/cleanup
index 78daece..c9a8d2b 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -5,6 +5,8 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
+# TODO: clean database, too
+
# we only clean if run interactive or if no one is logged in
if ! tty -s && \
[ -n "$(users)" ]; then
@@ -17,54 +19,24 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
exit
fi
-# remove blocked/broken/locked markes of packages not on the buildlist anymore
-
-{
- find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \
- sed '
- s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5|
- '
- awk '{
- print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken"
- print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked"
- print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked"
- }' "${work_dir}/build-list"
-} | \
- sort -k3 | \
- uniq -uf2 | \
- grep '^state ' | \
- awk '{print $2}' | \
- sed "s|^|${work_dir}/package-states/|" | \
- xargs -rn1 rm
-
# remove logs where package is not broken/locked anymore
{
- find "${build_log_directory}/error" -maxdepth 1 -type f -printf '1 %f %f\n' | \
- sed 's|\.[^. ]\+\.build-log\.gz$||'
- find "${work_dir}/package-states" -maxdepth 1 \( \
- -name '*.broken' -o \
- -name '*.done' -o \
- -name '*.locked' -o \
- -name '*.testing' \
- \) -printf '0 0 %f\n' | \
- sed 's|\.[^.]\+$||'
+ find "${build_log_directory}/error" -maxdepth 1 -type f -printf 'file %f\n'
+ # shellcheck disable=SC2016
+ printf 'SELECT "mysql",`failed_builds`.`log_file` FROM `failed_builds`;\n' | \
+ mysql_run_query | \
+ tr '\t' ' '
} | \
- sort -k3,3 -k1,2 | \
- uniq --group=prepend -f2 | \
- while read -r num file _; do
- if [ "${num}" = '0' ]; then
- while read -r line; do
- if [ -z "${line}" ]; then
- break
- fi
- done
- continue
- fi
- if [ -z "${num}${file}" ]; then
- continue
- fi
- rm -f "${build_log_directory}/error/${file}"
+ sort -k2,2 -k1,1 | \
+ uniq -uf 1 | \
+ sed -n '
+ s/^file //
+ T
+ p
+ ' | \
+ while read -r file; do
+ rm "${build_log_directory}/error/${file}"
done
# only keep 10 newest logs per failed package
@@ -97,14 +69,4 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \
-not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \
-delete
-# remove old package meta data
-delete_old_metadata
-
-# remove dependency graphs of packages on the deletion list
-sed '
- s|^|'"${webserver_directory}"'/graphs/|
- s|$|.png|
-' "${work_dir}/deletion-list" | \
- xargs -rn1 rm -f
-
exit 0
diff --git a/bin/common-functions b/bin/common-functions
deleted file mode 100755
index d5b6eaa..0000000
--- a/bin/common-functions
+++ /dev/null
@@ -1,1348 +0,0 @@
-#!/bin/sh
-
-# contains functions used by more than one script
-
-# TODO: include link depenendencies in run-depends metadata
-
-# shellcheck disable=SC2039
-
-if [ -z "${base_dir}" ]; then
- # just to make shellcheck happy
- . 'conf/default.conf'
-fi
-
-# find_pkgbuilds package repository git_repository git_revision mod_git_revision
-# find the PKGBUILD and modification of $package from $repository
-# sets $PKGBUILD and $PKGBUILD_mod
-
-find_pkgbuilds() {
-
- local package="$1"
- local repository="$2"
- local git_repository="$3"
- local git_revision="$4"
- local mod_git_revision="$5"
-
- local repo_path
- eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
- PKGBUILD=$(
- git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \
- tar -t 2> /dev/null | \
- grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \
- grep -v -- '-i686/PKGBUILD$' | \
- grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
- sort | \
- tail -n1
- )
-
- PKGBUILD_mod=$(
- git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \
- tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null
- ) || true
-
- if [ -z "${PKGBUILD}" ] && \
- [ -z "${PKGBUILD_mod}" ]; then
- >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \
- "${package}" \
- "${repository}" \
- "${git_repository}" \
- "${git_revision}" \
- "${mod_git_revision}"
- return 1
- fi
-
-}
-
-# find_repository_with_commit commit
-# find the repository which has $commit
-
-find_repository_with_commit() {
-
- local repository
-
- for repository in ${repo_names}; do
- # shellcheck disable=SC2016
- if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then
- echo "${repository}"
- return 0
- fi
- done
- >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1"
- exit 1
-
-}
-
-# find_git_repository_to_package_repository repository
-# find the git repository which tracks the package repository $repository
-
-find_git_repository_to_package_repository() {
-
- local repository
- local package_repository
- local repo_path
-
- package_repository="$1"
-
- if [ "$1" = 'build-support' ]; then
- echo 'packages'
- return 0
- fi
-
- for repository in ${repo_names}; do
- if [ "${repository}" = "archlinux32" ]; then
- continue
- fi
- eval 'repo_path="${repo_paths__'"${repository}"'}"'
- if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \
- tar -t --wildcards '*/repos' | \
- grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \
- cut -d/ -f3 | \
- sed 's|-[^-]\+$||' | \
- sort -u | \
- grep -qxF "${package_repository}"; then
- echo "${repository}"
- return 0
- fi
- done
- >&2 echo "can't find git repository with package repository '$1'"
- exit 1
-
-}
-
-# generate_package_metadata $package $git_revision $mod_git_revision $repository
-# or
-# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# generate the meta data files of a package (dependencies, built packages, ...)
-
-generate_package_metadata() {
-
- local package="$1"
- local git_revision="$2"
- local mod_git_revision="$3"
- local repository="$4"
- local file_prefix
- local file
- local PKGBUILD
-
- if [ $# -eq 1 ]; then
- # second form
- repository="${package##*.}"
- package="${package%.*}"
- mod_git_revision="${package##*.}"
- package="${package%.*}"
- git_revision="${package##*.}"
- package="${package%.*}"
- fi
-
- file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
-
- if [ -e "${file_prefix}.builds" ] && \
- [ -e "${file_prefix}.build-depends" ] && \
- [ -e "${file_prefix}.run-depends" ] && \
- [ -e "${file_prefix}.groups" ] && \
- [ -e "${file_prefix}.packages" ]; then
- return 0
- fi
-
- if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then
- printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"
- exit 1
- fi
- if [ ! -s "${file_prefix}.SRCINFO" ]; then
- >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO"
- exit 1
- fi
-
- # otherwise this just calls for trouble
- sed -i '
- /^[^=]*=\s*$/d
- s/_i686\(\s*=\)/\1/
- ' "${file_prefix}.SRCINFO"
-
- # extract "groups" = groups \cup provides
- grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u > \
- "${file_prefix}.groups"
-
- # extract "packages" = pkgname
- grep '^pkgname = ' "${file_prefix}.SRCINFO" | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u > \
- "${file_prefix}.packages"
-
- # extract "builds" = provides \cup pkgname \cup groups
- cat "${file_prefix}.groups" "${file_prefix}.packages" | \
- sort -u > \
- "${file_prefix}.builds"
-
- # extract "build-depends" = makedepends \cup checkdepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
- {
- {
- printf 'all_depend = %s\n' 'base' 'base-devel'
- sed -n "$(
- printf '/^\t%s = /p\n' \
- 'depends' \
- 'makedepends' \
- 'checkdepends'
- )" "${file_prefix}.SRCINFO"
- } | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u
- sed 'p' "${file_prefix}.builds"
- } | \
- sort | \
- uniq -u > \
- "${file_prefix}.build-depends"
-
- # extract "run-depends" = depends \cup \{ base \} \setminus "builds"
- {
- {
- printf 'all_depend = %s\n' 'base'
- sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO"
- } | \
- cut -d= -f2 | \
- sed 's|^\s\+||; s|[<>]$||' | \
- sort -u
- sed 'p' "${file_prefix}.builds"
- } | \
- sort | \
- uniq -u > \
- "${file_prefix}.run-depends"
-
- rm "${file_prefix}.SRCINFO"
-
-}
-
-# delete_old_metadata
-# delete old (=unneeded) meta data of packages
-
-delete_old_metadata() {
-
- local current_metadata
-
- current_metadata=$(
- find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
- sed '
- s|\.[^.]\+$||
- s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
- ' | \
- sort -u
- )
-
- ( # the new shell is intentional
- # what we have
- echo "${current_metadata}"
-
- # package-states should stay
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
- sed '
- s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
- ' | \
- sort -u | \
- sed 'p'
-
- # build-list items should stay
- sed 'p' "${work_dir}/build-list"
-
- tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
- echo "${current_metadata}" | \
- sort -k1,1 > \
- "${tmp_dir}/current-metadata"
-
- # the newest of the following should stay:
- {
- # deletion-list items
- cat "${work_dir}/deletion-list"
- # all packages in the repos
- for repo in ${repo_names}; do
- eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \
- tar -t | \
- sed '
- s|/$||
- /\//d
- '
- done
- } | \
- sort -u | \
- join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
- sort -k4,4 > \
- "${tmp_dir}/find-newest-revisions"
-
- uniq -uf3 < \
- "${tmp_dir}/find-newest-revisions" | \
- awk '{print $4 " " $1 " " $2 " " $3}' | \
- sed 'p'
-
- uniq -Df3 < \
- "${tmp_dir}/find-newest-revisions" | \
- uniq --group=append -f3 | \
- {
- revs=''
- mod_revs=''
- opkg=''
- orepo=''
- while read -r rev mod_rev repo pkg; do
-
- if [ -z "${rev}" ] && \
- [ -z "${mod_rev}" ] && \
- [ -z "${repo}" ] && \
- [ -z "${pkg}" ]; then
-
- printf '%s %s %s %s\n' \
- "$(
- printf '%s\n' ${revs} | \
- find_newest_of_git_revisions
- )" \
- "$(
- printf '%s\n' ${mod_revs} | \
- find_newest_of_git_revisions
- )" \
- "${orepo}" \
- "${opkg}"
-
- revs=''
- mod_revs=''
- orepo=''
- opkg=''
- continue
- fi
- revs=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${revs} ${rev} | \
- sort -u
- )
- mod_revs=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${mod_revs} ${mod_rev} | \
- sort -u
- )
- orepo="${repo}"
- opkg="${pkg}"
- done
- } | \
- awk '{print $4 " " $1 " " $2 " " $3}' | \
- sed 'p'
- ) | \
- sort | \
- uniq -u | \
- while read -r pkg rev mod_rev repo; do
- rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
- done
-}
-
-# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
-# print which (stable) repository a package belongs to
-
-repository_of_package() {
- local package="$1"
- local repository="${package##*.}"
- package="${package%.*}"
- local a32_rev="${package##*.}"
- package="${package%.*.*}"
-
- case "${repository}" in
- 'multilib')
- if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \
- tar -Ox | \
- grep -qFx "${package%.*.*.*}"; then
- echo 'extra'
- else
- echo 'community'
- fi
- ;;
- *)
- echo "${repository}"
- esac
-}
-
-# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending
-# print wether the specified package is an official package (print
-# $ending) or a community package (print 'community-$ending') or a
-# build-suppor package (print 'build-support')
-
-official_or_community() {
- local prepo
- prepo=$(repository_of_package "$1")
-
- if [ "${prepo}" = 'community' ]; then
- echo 'community-'"$2"
- elif [ "${prepo}" = 'build-support' ]; then
- echo 'build-support'
- else
- echo "$2"
- fi
-}
-
-# ls_master_mirror $path
-# list content of $path on the master mirror (via rsync)
-
-ls_master_mirror() {
-
- local path="$1"
-
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/${path}/" | \
- grep -v '\s\.$' | \
- awk '{print $5}'
-
-}
-
-# remove_old_package_versions $arch $repository $package_file
-
-# removes all older (not-newer) versions of $package_file
-# in all repositories not-older (newer) than $repository
-
-# TODO: should remove all other version (also newer) from
-# some repositories :-/
-
-# A package is considered not newer if
-# a) its version is not newer
-# A package is considered older if
-# b) its version is older or
-# c) if it's "not newer" and its architecture is 'any' and different or
-# d) if it's "not newer" and the other architecture is 'any' and different
-
-# this ensures an any package may replace arch-specific packages of the same version and vice versa
-
-remove_old_package_versions() {
-
- local arch="$1"
- local repository="$2"
- local package="$3"
-
- # repositories in which older packages should be deleted
- local delete_older_repositories
- # repositories in which not-newer packages should be deleted
- local delete_not_newer_repositories
-
- if echo "${standalone_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories="${repository}"
- delete_not_newer_repositories=''
-
- elif echo "${staging_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories="${repository}"
- delete_not_newer_repositories=$(
- echo "${staging_package_repositories}" | \
- grep -vxF "${repository}"
- ) || true
-
- elif echo "${testing_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories=$(
- printf '%s\n' "${staging_package_repositories}" "${repository}"
- )
- delete_not_newer_repositories=$(
- echo "${testing_package_repositories}" | \
- grep -vxF "${repository}"
- ) || true
-
- elif echo "${stable_package_repositories}" | \
- grep -qxF "${repository}"; then
-
- delete_older_repositories=$(
- printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}"
- )
- delete_not_newer_repositories=$(
- echo "${stable_package_repositories}" | \
- grep -vxF "${repository}"
- ) || true
-
- else
-
- >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}"
- return 1
-
- fi
-
- ( # the new shell is intentional
- tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
- {
- # the architecture of the package (any vs. i686)
- package_arch="${package##*-}"
- package_arch="${package_arch%%.*}"
- if [ "${package_arch}" = 'any' ]; then
- package_arch_regex_inverter='!'
- else
- unset package_arch_regex_inverter
- fi
-
- for repo in ${delete_older_repositories}; do
- ls_master_mirror "${arch}/${repo}" | \
- sed -n '
- /\.pkg\.tar\.xz$/!d
- s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
- /^'"$(str_to_regex "${package%-*-*-*}")"' / {
- s|^|2 '"${arch} ${repo}"' |
- / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{
- s|^2|0|
- }
- p
- }
- '
- done
- for repo in ${delete_not_newer_repositories}; do
- ls_master_mirror "${arch}/${repo}" | \
- sed -n '
- /\.pkg\.tar\.xz$/!d
- s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
- /^'"$(str_to_regex "${package%-*-*-*}")"' / {
- s|^|0 '"${arch} ${repo}"' |
- p
- }
- '
- done
- echo "${package%-*}" | \
- sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|'
-
- # the generated list contains the following columns:
- # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz
- } | \
- expand_version 5 | \
- sort -k5V,5 -k1n,1 | \
- shrink_version 5 | \
- sed -n '
- /^1 %cut% %it% %here% /q
- s/^[02] //
- s/ \(\S\+\)$/-\1/
- p
- ' | \
- sort -u > \
- "${tmp_dir}/packages-to-delete"
- # this file contains a list of packages to be deleted, one on each line:
- # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz
-
- cut -d' ' -f1,2 < \
- "${tmp_dir}/packages-to-delete" | \
- grep -vxF "${arch} ${repository}" | \
- sort -u > \
- "${tmp_dir}/repositories-to-modify"
-
- # fetch all databases being modified
- while read -r del_arch del_repo; do
- mkdir -p "${tmp_dir}/${del_arch}/${del_repo}"
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \
- "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \
- "${tmp_dir}/${del_arch}/${del_repo}/"
- done < \
- "${tmp_dir}/repositories-to-modify"
-
- while read -r del_arch del_repo del_package _; do
- if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then
- # we do not repo-remove the package in the target repository
- continue
- fi
- repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \
- "${del_package}"
- done < \
- "${tmp_dir}/packages-to-delete"
-
- # upload modified databases
- while read -r del_arch del_repo; do
- ${master_mirror_rsync_command} \
- "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \
- "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \
- "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/"
- done < \
- "${tmp_dir}/repositories-to-modify"
-
- # shellcheck disable=SC2016
- sed '
- s/\.pkg\.tar\.xz$//
- s/^\S\+ //
- s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/
- s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/
- s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/
- ' "${tmp_dir}/packages-to-delete" | \
- while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do
- printf 'DELETE FROM `binary_packages` WHERE'
- printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \
- 'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \
- 'repository' 'repositories' 'repositories' 'repositories' "$(printf '%s' "${repo}" | base64 -w0)"
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \
- 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
- 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
- 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
- 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \
- sed 's/ AND$//'
- printf ';\n'
- done | \
- ${mysql_command}
-
- sed '
- s| \(\S\+\)$|-\1|
- y| |/|
- s|^|rm "|
- s|$|"|
- p
- s|"$|.sig"|
- ' "${tmp_dir}/packages-to-delete" | \
- ${master_mirror_sftp_command}
- )
-
-}
-
-# wait_some_time $minimum $diff
-# wait between minimum and minimum+diff seconds (diff defaults to 30)
-
-wait_some_time() {
- local minimum=$1
- local diff=$2
- local random
-
- if [ -z "${diff}" ]; then
- diff=30
- fi
-
- random=$(
- dd if='/dev/urandom' count=1 2> /dev/null | \
- cksum | \
- cut -d' ' -f1
- )
-
- sleep $((minimum + random % diff))
-}
-
-# str_to_regex $string
-# escape dots for use in regex
-
-str_to_regex() {
- echo "$1" | \
- sed '
- s|[.[]|\\\0|g
- '
-}
-
-# make_source_info $package $repository $git_revision $mod_git_revision $output
-# create .SRCINFO from PKGBUILD within git repositories, output to $output
-
-make_source_info() {
-
- local package="$1"
- local repository="$2"
- local git_revision="$3"
- local mod_git_revision="$4"
- local output="$5"
-
- local git_repo
- local PKGBUILD
- local PKGBUILD_mod
-
- git_repo=$(find_repository_with_commit "${git_revision}")
-
- if [ -z "${git_repo}" ]; then
- return 1
- fi
-
- find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
-
- ( # the new shell is intentional
-
- tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX")
- trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
- extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0'
-
- {
- cd "${tmp_dir}"
- makepkg --printsrcinfo
- cd ..
- } |
- if [ "${package%-i18n}-i18n" = "${package}" ]; then
- sed '
- 1 a \\tdepends = '"${package%-i18n}"'
- '
- else
- cat
- fi > \
- "${output}"
-
- )
-
-}
-
-# recursively_umount_and_rm $dir
-# umount all mountpoints in $dir which are also in $dir's
-# filesystem, possibly also $dir itself and then
-# rm -rf --one-file-system $dir
-
-recursively_umount_and_rm() {
- local dir="$1"
-
- if [ -z "${dir}" ]; then
- >&2 echo 'ERROR: recursively_umount_and_rm requires an argument'
- exit 42
- fi
-
- find "${dir}" \
- -xdev -depth -type d \
- -exec 'mountpoint' '-q' '{}' ';' \
- -exec 'sudo' 'umount' '-l' '{}' ';'
- rm -rf --one-file-system "${dir}"
-}
-
-# mangle_pkgbuild $PKGBUILD [$sub_pkgrel]
-# mangle $arch in PKBUILDs to contain i486, i586, i686
-# append $sub_pkgrel to the pkgrel
-
-mangle_pkgbuild() {
- local PKGBUILD="$1"
- local sub_pkgrel="$2"
-
- if [ -n "${sub_pkgrel}" ]; then
- sub_pkgrel=".${sub_pkgrel}"
- fi
-
- sed -i '
- /^arch=[^#]*any/!{
- /^arch=(/s/(/(i686 /
- }
- s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/
- ' "${PKGBUILD}"
-}
-
-# find_newest_of_git_revisions
-# find newest git revision of the ones provided at stdin
-# (assuming linear history)
-
-find_newest_of_git_revisions() {
- local revisions
- local repo
- revisions=$(cat)
-
- if [ "$(
- echo "${revisions}" | \
- wc -l
- )" -eq 1 ]; then
-
- echo "${revisions}"
- return
-
- fi
-
- repo=$(
- find_repository_with_commit \
- "$(
- echo "${revisions}" | \
- grep -xm1 '[0-9a-f]\{40\}'
- )"
- )
-
- eval 'repo="${repo_paths__'"${repo}"'}"'
-
- echo "${revisions}" | \
- xargs -rn1 git -C "${repo}" rev-parse | \
- {
- newest=''
- while read -r current; do
- if [ -z "${newest}" ] || \
- git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then
- newest="${current}"
- fi
- done
- echo "${newest}"
- }
-}
-
-# find_package_repository_to_package $package $git_repository $git_commit
-# find the package repository a package from a given git repository
-# belongs to
-
-find_package_repository_to_package() {
-
- local package="$1"
- local git_repository="$2"
- local git_commit="$3"
- local repo_path
- local repo
-
- eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
- if [ "${git_repository}" = 'archlinux32' ]; then
- repo=$(
- git -C "${repo_path}" archive "${git_commit}" -- | \
- tar -t --wildcards "*/${package}/" | \
- cut -d/ -f1 | \
- sort -u
- )
- else
- repo=$(
- git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \
- tar -t | \
- cut -d/ -f3 | \
- grep -vxF '' | \
- grep -v 'staging\|testing\|-unstable' | \
- grep -v -- '-i686$' | \
- sed 's|-[^-]\+$||' | \
- sort -u
- )
- fi
-
- if [ -z "${repo}" ]; then
- return 1
- fi
-
- if [ "$(
- echo "${repo}" | \
- wc -l
- )" -ne 1 ]; then
- return 1
- fi
-
- echo "${repo}"
-
-}
-
-# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel
-# extract files found in the svn/git source directories
-# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly
-
-extract_source_directory() {
-
- local git_repo="$1"
- local rev="$2"
- local mod_rev="$3"
- local output="$4"
- local sub_pkgrel="$5"
-
- if [ -n "${PKGBUILD}" ]; then
- eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \
- tar -x --strip-components=3 -C "${output}"
- fi
-
- if [ -n "${PKGBUILD_mod}" ]; then
- git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \
- tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \
- true
- git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \
- tar -Ox "${PKGBUILD_mod}" >> \
- "${output}/PKGBUILD"
- fi
-
- mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}"
-
- # shellcheck disable=SC2016
- sed -i '/^\$Id\$$/d' "${output}/PKGBUILD"
-
-}
-
-# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository
-# return a list of packages on the build list which are (run- / build- / check-time)
-# dependencies of the given package
-
-find_dependencies_on_build_list() {
-
- local package="$1"
- local git_revision="$2"
- local mod_git_revision="$3"
- local repository="$4"
-
- generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
- {
- cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
- awk '{print $1 "." $2 "." $3 "." $4}' < \
- "${work_dir}/build-list" | \
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|\.builds|
- ' | \
- xargs -r cat | \
- sort -u
- } | \
- sort | \
- uniq -d
-
-}
-
-# download_sources_by_hash $package $repository $git_revision $git_mod_revision
-# try to download all sources by their hash into the current directory
-# returns 0 if any source was downloaded and 1 otherwise
-
-download_sources_by_hash() {
-
- local package="$1"
- local repository="$2"
- local git_revision="$3"
- local git_mod_revision="$4"
-
- local return_value=1
- local tmp_dir
- local sum_type
- local arch_suffix
-
- tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir)
-
- if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then
- >&2 echo 'download_sources_by_hash: make_source_info failed.'
- rm -rf --one-file-system "${tmp_dir}"
- return 1
- fi
-
- if ! [ -s "${tmp_dir}/.SRCINFO" ]; then
- >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.'
- rm -rf --one-file-system "${tmp_dir}"
- return 1
- fi
-
- for arch_suffix in '' '_i686'; do
- for sum_type in 'sha256sum' 'sha512sum'; do
- grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
- sed 's|^.* = ||' | \
- cat -n > \
- "${tmp_dir}/sums"
- grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
- sed '
- s|^.* = ||
- s|::.*$||
- s|.*/||
- ' | \
- cat -n > \
- "${tmp_dir}/urls"
- if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then
- join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \
- "${tmp_dir}/joined"
- while read -r sum file; do
- if [ "${sum}" = 'SKIP' ]; then
- continue
- fi
- if echo "${sum} ${file}" | \
- ${sum_type} -c > /dev/null 2>&1; then
- # the correct source is already there
- continue
- fi
- if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then
- mv "${tmp_dir}/transfer" "${file}"
- return_value=0
- fi
- done < \
- "${tmp_dir}/joined"
- fi
- done
- done
-
- rm -rf --one-file-system "${tmp_dir}"
- return ${return_value}
-
-}
-
-# expand_version $column_num
-# add "0:" to version in $colum_num-th column if no ":" is there (epoch)
-# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash)
-
-expand_version() {
- local column_num
- column_num="$1"
-
- sed '
- /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/
- /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/
- '
-}
-
-# shrink_version $column_num
-# remove "0:" from version in $colum_num-th column (epoch)
-# remove "+0" from version in $colum_num-th column (git count/hash)
-
-shrink_version() {
- local column_num
- column_num="$1"
-
- sed '
- s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/
- s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/
- '
-}
-
-# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ]
-
-# Return (to stdout) the biggest subset A of the packages in $omega whose
-# runtime dependencies in $omega \cup $keep are also in A
-
-# $all_builds and $all_depends either point to an empty file - then they will get
-# filled with cached data for subsequent calls - or to the same files of a previous
-# call
-
-# If non-empty, $force contains packages which are assumed to match the above
-# condition without checking.
-
-# The arguments are names of files with one $package.$revision.$mod_revision.$repository
-# per line.
-
-find_biggest_subset_of_packages() {
-
- ( # the new shell is intentional
- omega="$1"
- keep="$2"
- all_builds="$3"
- all_depends="$4"
- if [ $# -eq 4 ]; then
- force='/dev/null'
- elif [ $# -eq 5 ]; then
- force="$5"
- else
- >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#"
- return 2
- fi
-
- if [ ! -s "${all_builds}" ]; then
- find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \
- -exec sed '
- s|^|{} |
- s|^\S\+/||
- s|\.builds | |
- ' {} \; | \
- sort -k2,2 > \
- "${all_builds}"
- fi
-
- if [ ! -s "${all_depends}" ]; then
- find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \
- -exec sed '
- s|^|{} |
- s|^\S\+/||
- s|\.run-depends | |
- ' {} \; | \
- grep -v ' base$' | \
- sort -k2,2 > \
- "${all_depends}"
- fi
-
- sort -u "${omega}" | \
- sponge "${omega}"
-
- temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
- {
- sort -u "${keep}"
- cat "${force}" "${force}"
- } | \
- sort | \
- uniq -u > \
- "${temp_dir}/keep.new"
- touch "${temp_dir}/keep"
-
- while [ -s "${temp_dir}/keep.new" ]; do
- cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \
- sort -u | \
- sponge "${temp_dir}/keep"
-
- {
- # we append all packages which are run-dependencies of keep-packages
- # to the keep-list
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.run-depends|
- ' "${temp_dir}/keep" | \
- xargs -r grep -HF '' | \
- sed '
- s|^.*/||
- s|\.run-depends:| |
- ' | \
- sort -u | \
- sort -k2,2 | \
- uniq -f1 | \
- join -1 2 -2 2 -o 2.1 - "${all_builds}"
-
- # we append all packages with run-dependencies on the keep-list
- # to the keep-list
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.builds|
- ' "${temp_dir}/keep" | \
- xargs -r grep -HF '' | \
- sed '
- s|^.*/||
- s|\.builds:| |
- ' | \
- sort -u | \
- sort -k2,2 | \
- uniq -f1 | \
- join -1 2 -2 2 -o 2.1 - "${all_depends}"
- } | \
- sort -u | \
- join -1 1 -2 1 -o 2.1 - "${omega}" | \
- sort -u > \
- "${temp_dir}/keep.new"
-
- # "new" is only what has not been there before and what is not forced
- cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \
- sort | \
- uniq -u | \
- sponge "${temp_dir}/keep.new"
- done
-
- cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \
- sort | \
- uniq -u
-
- )
-
-}
-
-# sort_square_bracket_content $file
-# sort the content of [] in $file, print to stdout
-
-sort_square_bracket_content() {
- local file
- local line
- local token
- local token_list
- local rest
- file="$1"
-
- while read -r line; do
- printf '%s ' "${line}" | \
- tr ' ' '\n' | \
- while read -r token; do
- if echo "${token}" | \
- grep -qF '['; then
- printf '%s[' "${token%[*}"
- token="${token##*[}"
- token_list="${token%,}"
- while ! echo "${token_list}" | \
- grep -qF ']'; do
- read -r token
- token_list=$(
- printf '%s\n' \
- "${token_list}" \
- "${token%,}"
- )
- done
- rest="]${token_list#*]}"
- token_list="${token_list%%]*}"
- token=$(
- printf '%s' "${token_list}" | \
- sort | \
- sed 's|$|,|'
- printf '%s' "${rest}"
- )
- fi
- printf '%s\n' "${token}"
- done | \
- tr '\n' ' ' | \
- sed '
- s|, ]|]|g
- s| $||
- '
- printf '\n'
- done < \
- "${file}"
-}
-
-# smoothen_namcap_log $file
-# remove unneccesary differences from namcap-logs:
-# - remove architecture specific information
-# - sort lines
-# - sort content of square brackets
-
-smoothen_namcap_log() {
- local file
- file="$1"
- # shellcheck disable=SC2016
- sort_square_bracket_content "${file}" | \
- sed '
- # normalize architecture specific information
- s|i[34567]86|$ARCH|g
- s|x86\([-_]64\)\?|$ARCH|g
- # remove haskell hashes
- s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g
- ' | \
- sort | \
- sponge "${file}"
-}
-
-# print_list_of_archaic_packages $source1 $source2 ...
-# print a list of packages which have not been touched for a while,
-# but which are still in the pipeline, e.g. in $source1, $source2 or ...
-
-print_list_of_archaic_packages() {
- for source in "$@"; do
- case "${source}" in
- 'testing')
- # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \
- -exec head -n1 {} \; | \
- "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin
- # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated
- # and no longer block other packages from being moved
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \
- sed '
- s|\.testing$||
- '
- ;;
- 'build-list')
- while read -r pkg rev mod_rev repo; do
- git_repo=$(
- find_repository_with_commit "${rev}"
- )
- eval repo_path='"${repo_paths__'"${git_repo}"'}"'
- commit_date=$(
- git -C "${repo_path}" show -s --format=%ct "${rev}"
- )
- mod_commit_date=$(
- git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}"
- )
- if [ "${mod_commit_date}" -gt "${commit_date}" ]; then
- commit_date="${mod_commit_date}"
- fi
- # packages remaining longer than $max_package_age_build_list days on the build list
- if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then
- printf '%s %s %s %s\n' \
- "${pkg}" \
- "${rev}" \
- "${mod_rev}" \
- "${repo}"
- fi
- done < \
- "${work_dir}/build-list"
- ;;
- 'staging')
- # packages remaining longer than $max_package_age_staging days in staging
- find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \
- sed '
- s|\.done$||
- '
- ;;
- *)
- >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}"
- ;;
- esac
- done | \
- sort -u
-}
-
-# receive_buglist $category
-# receive the buglist from bugs.archlinux32.org for $category (e.g. 'Testing')
-# and print it to stdout
-
-receive_buglist() {
- local category
- category="$1"
- curl -LSs 'https://bugs.archlinux32.org/index.php?export_list=Export%20Tasklist' | \
- sed '
- 1 d
- /^[^,]\+,"Packages: '"${category}"'",/ ! d
- s/^\([^,]\+,\)\{3\}//
- s/^\("[^"]\+"\|[^"][^,]*\),"//
- s/".*$//
- '
-}
-
-# next_sub_pkgrel $package $git_revision $mod_git_revision $repository
-# giv out the next sub-pkgrel of the given package
-
-next_sub_pkgrel() {
- ( # new shell is intentional
- package="$1"
- git_revision="$2"
- mod_git_revision="$3"
- repository="$4"
- git_repo=$(find_git_repository_to_package_repository "${repository}")
-
- temp_dir=$(mktemp -d 'tmp.common-functions.next_sub_pkgrel.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
- find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
- extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${temp_dir}" '0'
-
- version=$(
- cd "${temp_dir}"
- makepkg --printsrcinfo | \
- sed -n '
- /^\s*\(epoch\|pkg\(name\|ver\|rel\)\) = /{s|^\s\+||;p}
- ' | \
- sed '
- /^pkgname = /{
- s/^.*= //
- w pkgnames
- d
- }
- s|^epoch = \(.*\)$|1 \1:|
- s|^pkgver = \(.*\)$|2 \1-|
- s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1|
- ' | \
- sort -k1n,1 | \
- sed '
- s|^[0-9] ||
- :a
- N
- s|\n[0-9] \(\S\+\)$|\1|
- ta
- '
- )
- sub_pkgrel=$(
- ls_master_mirror 'i686/*/' | \
- sed -n "$(
- sed '
- s/$/-'"${version}"'/
- s/\./\\./g
- s/^/^/
- s/$/\\(\\.\\([0-9]\\+\\)\\)\\?-[^-]\\+$/
- s|^.*$|/\0/{ s/\0/\\2/; s/^$/0/; p; b; }|
- ' "${temp_dir}/pkgnames"
- )" | \
- sort -n | \
- tail -n1
- )
- if [ -z "${sub_pkgrel}" ]; then
- echo '0'
- else
- echo "$((sub_pkgrel+1))"
- fi
- )
-}
-
-# modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
-# print the given modification revision possibly with a html link to github
-
-modification_revision_link() {
- local mod_rev="$1"
- local repo="$2"
- local pkg="$3"
-
- if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then
- printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \
- "${mod_rev}" \
- "${repo}" \
- "${pkg}" \
- "${mod_rev}"
- else
- printf '%s\n' \
- "${mod_rev}"
- fi
-}
-
-# trigger_mirror_refreshs
-# trigger a refresh of capable tier 1 mirrors (as backup for master mirror)
-
-trigger_mirror_refreshs() {
- local tmp_file
-
- tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir)
- date '+%s' > \
- "${tmp_file}"
- ${master_mirror_rsync_command} \
- "${tmp_file}" \
- "${master_mirror_rsync_directory}/lastupdate"
- rm "${tmp_file}"
- for trigger_url in ${mirror_refresh_trigger_urls}; do
- screen -S trigger-mirror-update -d -m curl -L "${trigger_url}"
- done
-}
diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
new file mode 100755
index 0000000..404a2b9
--- /dev/null
+++ b/bin/copy-to-build-support
@@ -0,0 +1,187 @@
+#!/bin/sh
+
+# copy the given package(s) into build-support
+
+# shellcheck source=conf/default.conf
+. "${0%/*}/../conf/default.conf"
+
+# shellcheck disable=SC2016
+usage() {
+ >&2 echo ''
+ >&2 echo 'copy-to-build-support [options] package-list:'
+ >&2 echo ' copy the packages listed in package-list into [build-support]'
+ >&2 echo ''
+ >&2 echo 'possible options:'
+ >&2 echo ' -h|--help:'
+ >&2 echo ' Show this help and exit.'
+ >&2 echo ' -w|--wait:'
+ >&2 echo ' Wait for lock if necessary.'
+ [ -z "$1" ] && exit 1 || exit "$1"
+}
+
+eval set -- "$(
+ getopt -o hw \
+ --long help \
+ --long wait \
+ -n "$(basename "$0")" -- "$@" || \
+ echo usage
+ )"
+
+wait_for_lock='-n'
+
+while true
+do
+ case "$1" in
+ -h|--help)
+ usage 0
+ ;;
+ -w|--wait)
+ wait_for_lock=''
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
+ exit 42
+ ;;
+ esac
+ shift
+done
+
+if [ "$#" -ne 1 ]; then
+ >&2 echo 'No package-list was given.'
+ usage
+fi
+
+exec 9> "${sanity_check_lock_file}"
+flock -s ${wait_for_lock} 9
+
+exec 8> "${package_database_lock_file}"
+flock ${wait_for_lock} 8
+
+tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture'
+sed -n '
+ s/.\+/\0 \0/
+ T
+ s/\.pkg\.tar\.xz$//
+ s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/
+ s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/
+ s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/
+ p
+' "$1" | \
+ while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do
+
+ # shellcheck disable=SC2016
+ id=$(
+ {
+ printf 'SELECT `binary_packages`.`id`,`repositories`.`name`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_architectures
+ printf ' WHERE'
+ printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
+ 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
+ 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
+ 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
+ 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \
+ 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)"
+ printf ' `architectures`.`name`=from_base64("%s")' \
+ "$(printf '%s' "${architecture}" | base64 -w0)"
+ printf ' LIMIT 1;\n'
+ } | \
+ mysql_run_query | \
+ tr '\t' ' '
+ )
+ if [ -z "${id}" ]; then
+ continue
+ fi
+ repository="${id#* }"
+ id="${id%% *}"
+
+ printf '%s\n' "${package}" >> \
+ "${tmp_dir}/packages"
+ for suffix in '' '.sig'; do
+ printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \
+ "${repository}" \
+ "${package}" \
+ "${suffix}" \
+ "${package}" \
+ "${suffix}"
+ done >> \
+ "${tmp_dir}/sftp-command"
+ printf '%s/i686/%s/%s\n' \
+ "${master_mirror_rsync_directory}" \
+ "${repository}" \
+ "${package}" | \
+ sed '
+ p
+ s/$/.sig/
+ ' >> \
+ "${tmp_dir}/to-copy"
+
+ # shellcheck disable=SC2016
+ {
+ printf 'INSERT IGNORE INTO `binary_packages`'
+ printf ' (`repository`'
+ # shellcheck disable=SC2086
+ printf ',`%s`' ${to_copy}
+ printf ')'
+ printf ' SELECT'
+ printf ' (SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="build-support")'
+ # shellcheck disable=SC2086
+ printf ',`binary_packages`.`%s`' ${to_copy}
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_architectures
+ printf ' WHERE'
+ printf ' `binary_packages`.`id`=%s;\n' \
+ "${id}"
+ } >> \
+ "${tmp_dir}/mysql-command"
+ done
+
+${master_mirror_rsync_command} \
+ "${master_mirror_rsync_directory}/i686/build-support/build-support.db."* \
+ "${master_mirror_rsync_directory}/i686/build-support/build-support.files."* \
+ "${tmp_dir}/"
+
+if [ -s "${tmp_dir}/to-copy" ]; then
+ mkdir "${tmp_dir}/transit/"
+ # shellcheck disable=SC2046
+ ${master_mirror_rsync_command} \
+ $(cat "${tmp_dir}/to-copy") \
+ "${tmp_dir}/transit/"
+ repo-add "${tmp_dir}/build-support.db.tar.gz" \
+ "${tmp_dir}/transit/"*".pkg.tar.xz"
+fi
+
+if [ -s "${tmp_dir}/sftp-command" ]; then
+ ${master_mirror_sftp_command} < \
+ "${tmp_dir}/sftp-command"
+fi
+
+${master_mirror_rsync_command} \
+ "${tmp_dir}/build-support.db."* \
+ "${tmp_dir}/build-support.files."* \
+ "${master_mirror_rsync_directory}/i686/build-support/"
+
+if [ -s "${tmp_dir}/mysql-command" ]; then
+ mysql_run_query < \
+ "${tmp_dir}/mysql-command"
+fi
+
+if [ -s "${tmp_dir}/packages" ]; then
+ while read -r package; do
+ remove_old_package_versions 'i686' 'build-support' "${package}"
+ done < \
+ "${tmp_dir}/packages"
+fi
+
+if [ -w "$1" ]; then
+ cat "${tmp_dir}/packages" > \
+ "$1"
+fi
diff --git a/bin/db-update b/bin/db-update
index fe45f41..6722f24 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -4,27 +4,13 @@
# additionally tested packages from testing to the respective stable
# repository (if possible [1])
-# 1] Condition for moving a package A from staging (testing) to
-# testing (stable) is that:
-# a) nothing on the build-list run-depends on A and
-# b) no done package B (in a not-more stable repository) which is
-# not being moved run-depends on A
+# The condition [1] is explained in the stored function
+# calculate_maximal_moveable_set which is created in bin/bootsrap-mysql
# TODO: separate locks for staging, testing (and stable)
-# TODO: handle deletion of parts of a split package
-
-# TODO: handle dependencies of parts of a split package separately
-
-# TODO: improve tracking of packages:
-# track packages in stable, too
-# know _exactly_ in which repository the packages are
-
-# TODO: fully integrate database (in other scripts too) - e.g. rely on
-# info in db instead of state-files
-
-# TODO: add possibility to manually clone a (binary) package (e.g.
-# for build-support) - this should also be possible via email
+# TODO: we should delete more packages than just the ones in repositories
+# where we move to (think of [extra] -> [community])
# shellcheck disable=SC2039
# shellcheck source=conf/default.conf
@@ -34,369 +20,40 @@
usage() {
>&2 echo ''
>&2 echo 'db-update [options] [packages]:'
- >&2 echo ' move possible packages from staging to testing.'
>&2 echo ' move tested packages from testing to stable.'
+ >&2 echo ' move possible packages from staging to testing.'
>&2 echo ''
>&2 echo 'possible options:'
- >&2 echo ' -b|--block: If necessary, wait for lock blocking.'
- >&2 echo ' -h|--help: Show this help and exit.'
- >&2 echo ' -n|--no-action: Only print what would be moved.'
- >&2 echo ' -s|--stabilize $package:'
- >&2 echo ' Assume, package $package can be stabilized.'
- >&2 echo ' -u|--unstage $package:'
- >&2 echo ' Assume, package $package can be unstaged.'
+ >&2 echo ' -b|--block:'
+ >&2 echo ' If necessary, wait for lock blocking.'
+ >&2 echo ' -f|--force $package-id:'
+ >&2 echo ' Force movement of Package with given id and move nothing else.'
+ >&2 echo ' -h|--help:'
+ >&2 echo ' Show this help and exit.'
+ >&2 echo ' -n|--no-action:'
+ >&2 echo ' Only print what would be moved.'
+ >&2 echo ' -p|--progressive:'
+ >&2 echo ' Move forward any package which replaces no package whose'
+ >&2 echo ' dependencies are all available somewhere.'
+ >&2 echo ' Note, that this _may_ move _less_ packages.'
[ -z "$1" ] && exit 1 || exit "$1"
}
-# move_packages file with one "$package $from_repository $to_repository" per line
-# the existence of a directory $tmp_dir is assumed
-
-move_packages() {
-
- if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
- >&2 echo 'move_packages: No tmp_dir provided.'
- exit 2
- fi
-
- local package
- local from_repo
- local to_repo
- local from_ending
- local rm_ending
- local to_ending
- local repo
- local part
- local dummynator
- local file
-
- if [ -e "${tmp_dir:?}/tmp" ]; then
- rm -rf --one-file-system "${tmp_dir:?}/tmp"
- fi
- mkdir "${tmp_dir}/tmp"
-
- touch "${tmp_dir}/tmp/repos"
- touch "${tmp_dir}/tmp/packages"
- touch "${tmp_dir}/tmp/master-mirror-listing"
- mkdir "${tmp_dir}/tmp/transit"
-
- if ${no_action}; then
- dummynator='echo'
- else
- dummynator=''
- fi
-
- ls_master_mirror 'i686' | \
- while read -r repo; do
- ls_master_mirror "i686/${repo}" | \
- sed "s|^|i686/${repo}/|" >> \
- "${tmp_dir}/tmp/master-mirror-listing"
- done
-
- while read -r package from_repo to_repo; do
- if [ -z "${package}" ]; then
- continue
- fi
-
- if ${no_action}; then
- printf \
- 'move "%s" from "%s" to "%s"\n' \
- "${package}" \
- "${from_repo}" \
- "${to_repo}"
- fi
-
- echo "${package}" >> \
- "${tmp_dir}/tmp/packages"
-
- if echo "${from_repo}" | \
- grep -q 'staging$' && \
- echo "${to_repo}" | \
- grep -q 'testing$'; then
- from_ending='done'
- rm_ending='test\(ing\|ed\)'
- to_ending='testing'
- elif echo "${from_repo}" | \
- grep -q 'testing$' && \
- ! echo "${to_repo}" | \
- grep -q 'testing$\|staging$'; then
- from_ending='tested'
- rm_ending=''
- to_ending=''
- else
- >&2 printf 'move_packages: Cannot move package "%s" from "%s" to "%s".\n' "${package}" "${from_repo}" "${to_repo}"
- exit 2
- fi
-
- echo "${from_repo}" > \
- "${tmp_dir}/tmp/${package}.from_repo"
- echo "${to_repo}" > \
- "${tmp_dir}/tmp/${package}.to_repo"
- echo "${from_ending}" > \
- "${tmp_dir}/tmp/${package}.from_ending"
- echo "${rm_ending}" > \
- "${tmp_dir}/tmp/${package}.rm_ending"
- echo "${to_ending}" > \
- "${tmp_dir}/tmp/${package}.to_ending"
-
- if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
- >&2 printf 'move_packages: Cannot find package state file "%s"\n' "${package}.${from_ending}"
- exit 2
- fi
-
- cp \
- "${work_dir}/package-states/${package}.${from_ending}" \
- "${tmp_dir}/tmp/${package}.parts"
-
- sed \
- 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
- "${tmp_dir}/tmp/${package}.parts" > \
- "${tmp_dir}/tmp/${package}.parts_names"
-
- sed \
- 'p;s|$|.sig|' \
- "${tmp_dir}/tmp/${package}.parts" > \
- "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
- while read -r part; do
- if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/tmp/master-mirror-listing"; then
- >&2 printf \
- 'move_packages: Cannot find file "%s", part of package "%s".\n' \
- "i686/${from_repo}/${part}" \
- "${package}"
- exit 2
- fi
- done < \
- "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
- mkdir -p "${tmp_dir}/tmp/${from_repo}"
- mkdir -p "${tmp_dir}/tmp/${to_repo}"
-
- repos=$(
- # shellcheck disable=SC2046
- printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/tmp/repos") | \
- sort -u
- )
- echo "${repos}" > \
- "${tmp_dir}/tmp/repos"
-
- done < \
- "$1"
-
- if ${no_action}; then
- find "${tmp_dir}/tmp" -type f | \
- while read -r file; do
- if [ "${file%.pkg.tar.xz}.pkg.tar.xz" = "${file}" ] ||
- [ "${file%.pkg.tar.xz.sig}.pkg.tar.xz.sig" = "${file}" ]; then
- echo "'${file}'"
- else
- echo "${file}:"
- sed 's|^|<<|;s|$|>>|' "${file}"
- fi
- echo
- done
- fi
-
- # receive the *.db.tar.gz's and *.files.tar.gz's
-
- while read -r repo; do
-
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
- "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
- "${tmp_dir}/tmp/${repo}/"
-
- # add and remove the packages locally
-
- if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".from_repo"; then
-
- # shellcheck disable=SC2046
- repo-remove -q \
- "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
- $(
- grep -lxF "${repo}" "${tmp_dir}/tmp/"*".from_repo" | \
- sed '
- s|\.from_repo$|.parts_names|
- ' | \
- xargs -rn1 cat
- )
- fi
-
- if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".to_repo"; then
- grep -lxF "${repo}" "${tmp_dir}/tmp/"*".to_repo" | \
- sed '
- s|\.to_repo$||
- ' | \
- while read -r package; do
- while read -r part; do
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}" \
- "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}.sig" \
- "${tmp_dir}/tmp/transit/"
- repo-add -q \
- "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
- "${tmp_dir}/tmp/transit/${part}"
- rm \
- "${tmp_dir}/tmp/transit/${part}" \
- "${tmp_dir}/tmp/transit/${part}.sig"
- done < \
- "${package}.parts"
- done
- fi
-
- done < "${tmp_dir}/tmp/repos"
-
- if ${no_action}; then
- find "${tmp_dir}/tmp" -type f
- fi
-
- # move the packages remotely via sftp
-
- {
- while read -r package; do
-
- if [ -z "${package}" ]; then
- continue
- fi
-
- while read -r part; do
- if [ -z "${part}" ]; then
- continue
- fi
- printf \
- 'rename "%s" "%s"\n' \
- "i686/$(cat "${tmp_dir}/tmp/${package}.from_repo")/${part}" \
- "i686/$(cat "${tmp_dir}/tmp/${package}.to_repo")/${part}"
- done < \
- "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
- done < \
- "${tmp_dir}/tmp/packages"
- echo 'quit'
- } | \
- if ${no_action}; then
- sed 's|^|sftp: |'
- else
- ${master_mirror_sftp_command}
- fi
-
- # and push our local *.db.tar.gz via rsync
-
- while read -r repo; do
-
- # shellcheck disable=SC2086
- ${dummynator} ${master_mirror_rsync_command} \
- "${tmp_dir}/tmp/${repo}/${repo}.db."* \
- "${tmp_dir}/tmp/${repo}/${repo}.files."* \
- "${master_mirror_rsync_directory}/i686/${repo}/"
-
- done < \
- "${tmp_dir}/tmp/repos"
-
- while read -r package; do
-
- # then we can safely remove old versions
-
- while read -r part; do
- ${dummynator} remove_old_package_versions 'i686' "$(cat "${tmp_dir}/tmp/${package}.to_repo")" "${part}"
- done < \
- "${tmp_dir}/tmp/${package}.parts"
-
- # and update the state files
-
- from_ending=$(
- cat "${tmp_dir}/tmp/${package}.from_ending"
- )
- rm_ending=$(
- cat "${tmp_dir}/tmp/${package}.rm_ending"
- )
- to_ending=$(
- cat "${tmp_dir}/tmp/${package}.to_ending"
- )
-
- if [ -z "${to_ending}" ]; then
- ${dummynator} rm \
- "${work_dir}/package-states/${package}.${from_ending}"
- else
- # remove old state files of $package with ending $rm_ending
- find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
- -regex '.*/'"$(str_to_regex "${package%.*.*.*}")"'\(\.[^.]\+\)\{3\}\.'"${rm_ending}" \
- -execdir ${dummynator} rm {} \;
- ${dummynator} mv \
- "${work_dir}/package-states/${package}.${from_ending}" \
- "${work_dir}/package-states/${package}.${to_ending}"
- fi
-
- done < \
- "${tmp_dir}/tmp/packages"
-
- # shellcheck disable=SC2016
- while read -r package; do
- while read -r part; do
- printf 'UPDATE `binary_packages`'
- printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
- printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
- printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s"))' \
- "$(
- base64_encode_each < \
- "${tmp_dir}/tmp/${package}.to_repo"
- )"
- printf ' WHERE'
- printf ' `repositories`.`name`=from_base64("%s")' \
- "$(
- base64_encode_each < \
- "${tmp_dir}/tmp/${package}.from_repo"
- )"
- printf '%s\n' "${part}" | \
- sed '
- s/\.pkg\.tar\.xz$//
- s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/
- s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
- s/^\(.\+\)-\([^-:]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/\1\n\2\n\3\n\4\n\5\n\6/
- ' | \
- base64_encode_each | \
- tr '\n' ' ' | \
- sed '
- s,\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) $,'"$(
- printf ' AND `binary_packages`.`%s`=from_base64("%s")' \
- 'pkgname' '\1' \
- 'epoch' '\2' \
- 'pkgver' '\3' \
- 'pkgrel' '\4' \
- 'sub_pkgrel' '\5'
- printf ' AND `architectures`.`name`=from_base64("\\6")'
- )"',
- '
- printf ';\n'
- done < \
- "${tmp_dir}/tmp/${package}.parts"
- done < \
- "${tmp_dir}/tmp/packages" | \
- if ${no_action}; then
- sed 's|^|mysql: |'
- else
- ${mysql_command}
- fi
-
- ${dummynator} trigger_mirror_refreshs
-
- rm -rf --one-file-system "${tmp_dir:?}/tmp"
-
-}
-
eval set -- "$(
- getopt -o bf:hns:u: \
+ getopt -o bf:hnp \
--long block \
- --long from: \
+ --long force \
--long help \
--long no-action \
- --long stabilize: \
- --long unstage: \
+ --long progressive \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
block_flag='-n'
no_action=false
+progressive=false
+force_ids=''
while true
do
@@ -404,19 +61,22 @@ do
-b|--block)
block_flag=''
;;
+ -f|--force)
+ shift
+ force_ids=$(
+ printf '%s' "$1" | \
+ base64 -w0
+ printf '\n%s' "${force_ids}"
+ )
+ ;;
-h|--help)
usage 0
;;
-n|--no-action)
no_action=true
;;
- -s|--stabilze)
- shift
- packages_to_force_stabilize="${packages_to_force_stabilize} $1"
- ;;
- -u|--unstage)
- shift
- packages_to_force_unstage="${packages_to_force_unstage} $1"
+ -p|--progressive)
+ progressive=true
;;
--)
shift
@@ -435,237 +95,306 @@ if [ $# -ne 0 ]; then
usage
fi
+if ${progressive} && \
+ [ -n "${force_ids}" ]; then
+ >&2 echo 'db-update: conflicting arguments'
+ usage
+fi
+
if [ -s "${work_dir}/build-master-sanity" ]; then
>&2 echo 'Build master is not sane.'
exit
fi
-tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-for package in ${packages_to_force_stabilize}; do
- # some sanity checks
- if [ ! -f "${work_dir}/package-states/${package}.tested" ] && \
- [ ! -f "${work_dir}/package-states/${package}.testing" ] && \
- [ ! -f "${work_dir}/package-states/${package}.done" ] && \
- ! tr ' ' '.' < \
- "${work_dir}/build-list" | \
- grep -qxF "${package}"; then
- >&2 printf 'Package "%s" is not in testing, staging or on the build list!\n' "${package}"
- exit 2
- fi
-done
+# Create tmp_dir, lock and trap.
-for package in ${packages_to_force_unstage}; do
- # some sanity checks
- if [ ! -f "${work_dir}/package-states/${package}.done" ] && \
- ! tr ' ' '.' < \
- "${work_dir}/build-list" | \
- grep -qxF "${package}"; then
- >&2 printf 'Package "%s" is not in staging or on the build list!\n' "${package}"
- exit 2
- fi
-done
-
-# Create a lock file and a trap.
-
-exec 9> "${build_list_lock_file}"
+exec 9> "${package_database_lock_file}"
if ! flock ${block_flag} 9; then
- >&2 echo 'come back (shortly) later - I cannot lock build list.'
- exit 0
-fi
-
-exec 8> "${package_database_lock_file}"
-if ! flock ${block_flag} 8; then
>&2 echo 'come back (shortly) later - I cannot lock package database.'
exit 0
fi
-exec 7> "${sanity_check_lock_file}"
-if ! flock -s ${block_flag} 7; then
+exec 8> "${sanity_check_lock_file}"
+if ! flock -s ${block_flag} 8; then
>&2 echo 'come back (shortly) later - sanity-check currently running.'
exit 0
fi
-clean_up_lock_file() {
- rm -f "${package_database_lock_file}" "${build_list_lock_file}"
- rm -rf --one-file-system "${tmp_dir}"
-}
-
-trap clean_up_lock_file EXIT
-
-# sanity check
-
-for ending in 'done' 'tested'; do
- errors=$(
- find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
- sed 's|\(\.[^.]\+\)\{4\}$||' | \
- sort | \
- uniq -d
- )
- if [ -n "${errors}" ]; then
- >&2 echo 'Removing duplicates not yet implemented:'
- >&2 echo "${errors}"
- exit 42
- fi
-done
-
-# packages which are done
+tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
- sed '
- s|\.done$||
- ' | \
- sort -u > \
- "${tmp_dir}/done-packages"
+export TMPDIR="${tmp_dir}"
-# packages still on the build-list
+for source_stability in 'testing' 'staging'; do
+ find "${tmp_dir}" -mindepth 1 -delete
-grep -vxF 'break_loops' "${work_dir}/build-list" | \
- tr ' ' '.' | \
- sort -u > \
- "${tmp_dir}/build-list-packages"
+ # shellcheck disable=SC2016
+ {
+ if [ -n "${force_ids}" ]; then
+ printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \
+ 'moveable' 'replaced'
+ printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+ printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+ printf ' VALUES'
+ # shellcheck disable=SC2086
+ printf '(from_base64("%s"),NULL),' \
+ ${force_ids} | \
+ sed 's/,$/;\n/'
+ printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' WHERE `repository_stabilities`.`name`!="%s";\n' \
+ "${source_stability}"
+ printf 'UPDATE `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
+ printf ' SET `moveable_binary_packages`.`to_repository`=`repository_moves`.`to_repository`;\n'
+ printf 'DELETE FROM `moveable_binary_packages` WHERE `moveable_binary_packages`.`to_repository` IS NULL;\n'
+ printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+ printf ' SELECT `binary_packages`.`id`,`moveable_binary_packages`.`id`'
+ printf ' FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` AS `subst_bp` ON `moveable_binary_packages`.`id`=`subst_bp`.`id`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
+ printf ' AND `binary_packages`.`repository`=`moveable_binary_packages`.`to_repository`;\n'
+ elif ${progressive}; then
+ printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \
+ 'moveable' 'replaced'
+ printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+
+ printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+ printf ' SELECT `binary_packages`.`id`,`subst_bp`.`id`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`to_repository`=`binary_packages`.`repository`'
+ printf ' JOIN `binary_packages` AS `subst_bp`'
+ printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
+ printf ' AND `repository_moves`.`from_repository`=`subst_bp`.`repository`'
+ mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+ mysql_join_repositories_repository_stabilities 'subst_r' 'subst_rs'
+ printf ' AND `subst_rs`.`name`="%s"' \
+ "${source_stability}"
+ mysql_join_binary_packages_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `install_target_providers`'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ');\n'
+
+ printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+ printf ' SELECT `replaced_binary_packages`.`replaced_by`,`binary_packages`.`repository`'
+ printf ' FROM `replaced_binary_packages`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`'
+ printf ';\n'
-find "${work_dir}/package-infos" -name '*.groups' \
- -exec grep -qxF 'base' {} \; \
- -printf '%f\n' | \
- sed '
- s|\.groups$||
- ' | \
- sort -u > \
- "${tmp_dir}/base-packages"
+ printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+ printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ mysql_join_repositories_repository_stabilities
+ printf ' AND `repository_stabilities`.`name`="%s"' \
+ "${source_stability}"
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `binary_packages` AS `repl_bp`'
+ printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`'
+ printf ' AND `repl_bp`.`repository`=`repository_moves`.`to_repository`'
+ printf ');\n'
+ else
+ printf 'CALL calculate_maximal_moveable_set("%s");\n' \
+ "${source_stability}"
+ fi
-{
- # shellcheck disable=SC2086
- printf '%s\n' ${packages_to_force_unstage}
- print_list_of_archaic_packages 'build-list'
-} > \
- "${tmp_dir}/force-unstage-packages"
-
->&2 printf 'calculate what packages should be unstaged ...'
-find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-unstage-packages" > \
- "${tmp_dir}/unstage-packages"
->&2 printf ' ok.\n'
-
-# no base packages on the build list anymore?
-if [ -z "$(
- join -j 1 \
- "${tmp_dir}/base-packages" \
- "${tmp_dir}/build-list-packages"
- )" ]; then
-
- >&2 echo 'db-update unstage: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages'
- for s in "${tmp_dir}/all-builds" "${tmp_dir}/all-depends"; do
- sed '/ base$/d' "${s}" > \
- "${s}.no-base"
- done
+ printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `moveable_binary_packages`.`to_repository`'
+ printf ' FROM `moveable_binary_packages`;\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `binary_packages`.`repository`'
+ printf ' FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`;\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `binary_packages`.`repository`'
+ printf ' FROM `replaced_binary_packages`'
+ printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`;\n'
+
+ printf 'SELECT "repositories",`repositories`.`name`'
+ printf ' FROM `repositories`'
+ printf ' JOIN `rps` ON `rps`.`id`=`repositories`.`id`;\n'
+
+ printf 'SELECT "mv.id",`moveable_binary_packages`.`id`,`moveable_binary_packages`.`to_repository`'
+ printf ' FROM `moveable_binary_packages`;\n'
+
+ printf 'SELECT "mv",'
+ mysql_package_name_query
+ printf ',`repositories`.`name`,`new_repo`.`name`'
+ printf ' FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_architectures
+ printf ' JOIN `repositories` AS `new_repo` ON `new_repo`.`id`=`moveable_binary_packages`.`to_repository`'
+ printf ';\n'
+
+ printf 'SELECT "rm.id",`replaced_binary_packages`.`id`'
+ printf ' FROM `replaced_binary_packages`;\n'
+
+ printf 'SELECT "rm",'
+ mysql_package_name_query
+ printf ',`repositories`.`name`'
+ printf ' FROM `replaced_binary_packages`'
+ printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_architectures
+ printf ';\n'
+ } | \
+ mysql_run_query | \
+ tr '\t' ' ' | \
+ grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
+ while read -r what content; do
+ printf '%s\n' "${content}" >> \
+ "${tmp_dir}/${what}"
+ done
- >&2 printf 'calculate what packages should be unstaged ...'
- find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-unstage-packages" > \
- "${tmp_dir}/unstage-packages"
- >&2 printf ' ok.\n'
+ if [ ! -s "${tmp_dir}/repositories" ]; then
+ >&2 printf 'Nothing to move from %s.\n' "${source_stability}"
+ continue
+ fi
-fi
+ touch \
+ "${tmp_dir}/mv" \
+ "${tmp_dir}/mv.id" \
+ "${tmp_dir}/rm" \
+ "${tmp_dir}/rm.id"
-{
# shellcheck disable=SC2086
- printf '%s\n' ${packages_to_force_stabilize}
- print_list_of_archaic_packages 'build-list' 'staging' 'testing'
-} > \
- "${tmp_dir}/force-stabilize-packages"
-
-# calculate what packages should be stabilized
-
-{
- cat "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages"
- find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.testing' -printf '%f\n' | \
- sed 's|\.testing$||'
-} | \
- sort -u > \
- "${tmp_dir}/keep-packages"
-
-find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.tested' -printf '%f\n' | \
- sed 's|\.tested$||' > \
- "${tmp_dir}/stabilize-packages"
-
-# no base packages on the build list or in staging anymore?
-if [ -z "$(
- cat \
- "${tmp_dir}/build-list-packages" \
- "${tmp_dir}/done-packages" | \
- sort -u | \
- join -j 1 \
- "${tmp_dir}/base-packages" \
- -
- )" ]; then
-
- >&2 echo 'db-update stabilize: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages'
- >&2 printf 'calculate what packages should be stabilized ...'
- find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-stabilize-packages" | \
- sponge "${tmp_dir}/stabilize-packages"
- >&2 printf ' ok.\n'
-
-else
-
- >&2 printf 'calculate what packages should be stabilized ...'
- find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-stabilize-packages" | \
- sponge "${tmp_dir}/stabilize-packages"
- >&2 printf ' ok.\n'
-
-fi
+ for s in "${tmp_dir}/"*; do
+ sort -u "${s}" | \
+ sponge "${s}"
+ done
-# unlock build list
+ # receive the repository databases from the master mirror
+ mkdir "${tmp_dir}/dbs"
+ while read -r repo; do
+ mkdir "${tmp_dir}/dbs/${repo}"
+ # shellcheck disable=SC2086
+ ${master_mirror_rsync_command} \
+ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
+ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
+ "${tmp_dir}/dbs/${repo}/"
+ tar -Oxzf "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz" --wildcards '*/desc' | \
+ sed -n '
+ /^%FILENAME%$/{
+ N
+ s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/
+ T
+ p
+ }
+ '
+ done < \
+ "${tmp_dir}/repositories" | \
+ while read -r old new; do
+ for file in 'rm' 'mv'; do
+ sed -i '
+ s/\(\s\|^\)'"$(str_to_regex "${old}")"'\(\s\|$\)/\1'"${new}"'\2/
+ ' "${tmp_dir}/${file}"
+ done
+ done
-rm -f "${build_list_lock_file}"
-flock -u 9
+ # remove to-be-deleted packages
+ # shellcheck disable=SC2094
+ cut -d' ' -f2 < \
+ "${tmp_dir}/rm" | \
+ sort -u | \
+ while read -r repo; do
+ grep " $(str_to_regex "${repo}")\$" "${tmp_dir}/rm" | \
+ sed '
+ s/\(-[^-]\+\)\{3\} \S\+$//
+ ' | \
+ xargs -r repo-remove -q "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz"
+ done
-clean_up_lock_file() {
- rm -rf --one-file-system "${tmp_dir}"
- rm -f "${package_database_lock_file}"
-}
+ # copy and delete moved packages
+ # shellcheck disable=SC2094
+ cut -d' ' -f2,3 < \
+ "${tmp_dir}/mv" | \
+ sort -u | \
+ while read -r from_repo to_repo; do
+ grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \
+ sed '
+ s/-[^-]\+ \S\+ \S\+$//
+ ' | \
+ xargs -r "${base_dir}/bin/repo-copy" \
+ "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" \
+ "${tmp_dir}/dbs/${to_repo}/${to_repo}.db.tar.gz"
+ grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \
+ sed '
+ s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
+ ' | \
+ xargs -r repo-remove -q \
+ "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz"
+ done
-# testing -> stable
+ # move the packages remotely via sftp
+ {
+ sed '
+ s,^\(\S\+\) \(\S\+\)$,rm "i686/\2/\1"\nrm "i686/\2/\1.sig",
+ ' "${tmp_dir}/rm"
+ sed '
+ s,^\(\S\+\) \(\S\+\) \(\S\+\)$,rename "i686/\2/\1" "i686/\3/\1"\nrename "i686/\2/\1.sig" "i686/\3/\1.sig",
+ ' "${tmp_dir}/mv"
+ echo 'quit'
+ } | \
+ if ${no_action}; then
+ sed 's|^|sftp: |'
+ else
+ ${master_mirror_sftp_command}
+ fi
-while read -r package; do
- if [ -z "${package}" ]; then
- continue
- fi
- printf '%s %s %s\n' \
- "${package}" \
- "$(official_or_community "${package}" 'testing')" \
- "$(repository_of_package "${package}")"
-done < \
- "${tmp_dir}/stabilize-packages" | \
- sponge "${tmp_dir}/stabilize-packages"
-
-# staging -> testing
-
-while read -r package; do
- if [ -z "${package}" ]; then
+ if ${no_action}; then
continue
fi
- printf '%s %s %s\n' \
- "${package}" \
- "$(official_or_community "${package}" 'staging')" \
- "$(official_or_community "${package}" 'testing')"
-done < \
- "${tmp_dir}/unstage-packages" | \
- sponge "${tmp_dir}/unstage-packages"
-
-if [ -s "${tmp_dir}/stabilize-packages" ]; then
- >&2 printf 'move packages from *testing/ to the stable repos ...'
- move_packages "${tmp_dir}/stabilize-packages"
- >&2 printf ' ok.\n'
-else
- >&2 printf 'Nothing to move from *testing to the stable repos.\n'
-fi
-if [ -s "${tmp_dir}/unstage-packages" ]; then
- >&2 printf 'move packages from *staging to *testing ...'
- move_packages "${tmp_dir}/unstage-packages"
- >&2 printf ' ok.\n'
-else
- >&2 printf 'Nothing to move from *staging to *testing.\n'
-fi
+ # and push our local *.db.tar.gz via rsync
+ while read -r repo; do
+ # shellcheck disable=SC2086
+ ${master_mirror_rsync_command} \
+ "${tmp_dir}/dbs/${repo}/${repo}.db."* \
+ "${tmp_dir}/dbs/${repo}/${repo}.files."* \
+ "${master_mirror_rsync_directory}/i686/${repo}/"
+ done < \
+ "${tmp_dir}/repositories"
+
+ # shellcheck disable=SC2016
+ {
+ printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moved_binary_packages` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
+ "${tmp_dir}/mv.id" 'moved_binary_packages' \
+ "${tmp_dir}/rm.id" 'replaced_binary_packages'
+ printf 'DELETE `binary_packages` FROM `binary_packages`'
+ printf ' JOIN `replaced_binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`;\n'
+ printf 'UPDATE `binary_packages`'
+ printf ' JOIN `moved_binary_packages` ON `binary_packages`.`id`=`moved_binary_packages`.`id`'
+ printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n'
+ } | \
+ mysql_run_query
+done
-clean_up_lock_file
+trigger_mirror_refreshs
diff --git a/bin/delete-packages b/bin/delete-packages
index 26d323a..4343bc7 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -13,7 +13,7 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
-# TODO: finish this
+# TODO: delete other to-be-deleted packages if asked to do so
# shellcheck disable=SC2016
usage() {
@@ -69,18 +69,15 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
exit
fi
-tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX")
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
# Create a lock file and a trap.
if ! ${no_action}; then
- exec 9> "${build_list_lock_file}"
- if ! flock ${block_flag} 9; then
- >&2 echo 'come back (shortly) later - I cannot lock build list.'
- exit 0
- fi
+# exec 9> "${build_list_lock_file}"
+# if ! flock ${block_flag} 9; then
+# >&2 echo 'come back (shortly) later - I cannot lock build list.'
+# exit 0
+# fi
exec 8> "${package_database_lock_file}"
if ! flock ${block_flag} 8; then
@@ -96,260 +93,158 @@ if ! ${no_action}; then
fi
-clean_up_lock_file() {
- if ! ${no_action}; then
- rm -f "${package_database_lock_file}" "${build_list_lock_file}"
- fi
- rm -rf --one-file-system "${tmp_dir}"
-}
-
-trap clean_up_lock_file EXIT
-
-cp \
- "${work_dir}/deletion-list" \
- "${work_dir}/build-list" \
- "${tmp_dir}/"
-
-all_repos="${standalone_package_repositories} ${stable_package_repositories} ${testing_package_repositories} ${staging_package_repositories}"
-all_repos="core"
-
-for repo in ${all_repos}; do
- mkdir "${tmp_dir}/${repo}"
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
- "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
- "${tmp_dir}/${repo}/"
-done
-
-for repo in ${all_repos}; do
- tar -C "${tmp_dir}/${repo}" -xzf "${tmp_dir}/${repo}/${repo}.db.tar.gz"
-done
+tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-find "${tmp_dir}" -mindepth 3 -maxdepth 3 -name 'desc' -exec \
+# shellcheck disable=SC2016
+{
+ printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'INSERT IGNORE INTO `to_deletes`'
+ printf ' SELECT `binary_packages`.`id`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ printf ' WHERE `binary_packages`.`is_to_be_deleted`'
+ printf ' AND NOT EXISTS ('
+ printf 'SELECT * FROM `install_target_providers`'
+ mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+ mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+ mysql_join_repositories_repository_stabilities 'prov_r'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ' AND `repository_stabilities`.`name` NOT IN ("forbidden","virtual")'
+ printf ');\n'
+
+ printf ' SELECT DISTINCT "repo",`repositories`.`name`'
+ printf ' FROM `to_deletes`'
+ printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ printf ';\n'
+
+ printf ' SELECT "package",`repositories`.`name`,`binary_packages`.`pkgname`'
+ printf ' FROM `to_deletes`'
+ printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ printf ';\n'
+
+ printf ' SELECT "package-file",`repositories`.`name`,'
+ mysql_package_name_query
+ printf ' FROM `to_deletes`'
+ printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_architectures
+ printf ';\n'
+
+ printf ' SELECT "package-id",`to_deletes`.`id`'
+ printf ' FROM `to_deletes`'
+ printf ';\n'
+} | \
+ mysql_run_query | \
sed '
- /^%DEPENDS%$/{
- s/.*//
- :dep_loop
- N
- /\n$/{
- s@\(^\|\n\)\(.\)@\1depends \2@g
- bend
- }
- bdep_loop
+ y/\t/ /
+ /^repo /{
+ s/^\S\+ //
+ w '"${tmp_dir}"'/repositories
+ d
}
- /^%PROVIDES%$/{
- s/.*//
- :pro_loop
- N
- /\n$/{
- s@\(^\|\n\)\(.\)@\1provides \2@g
- bend
- }
- bpro_loop
+ /^package /{
+ s/^\S\+ //
+ w '"${tmp_dir}"'/packages
+ d
}
- d
- :end
- s/[<>=]\S*\($\|\n\)/\1/g
- s#\(^\|\n\)\(.\)#\1{} \2#g
- ' {} \; \
- -printf '%p provides %p\n' | \
- sed '
- /^$/d
- s|^\S\+/\([^/ ]\+\)/\([^/ ]\+\)\(-[^-/ ]\+\)\{2\}/desc |\1 \2 |
- s| \S\+/\([^/]\+\)\(-[^/-]\+\)\{2\}/desc$| \1|
- ' | \
- awk '{print $3 " " $1 " " $2 " " $4}' | \
- sed -n '
- /^provides /{
- s/^provides //
- w '"${tmp_dir}"'/db.provides
- b
+ /^package-file /{
+ s/^\S\+ //
+ s, ,/,
+ w '"${tmp_dir}"'/package-files
+ s/$/.sig/
+ w '"${tmp_dir}"'/package-files
+ d
}
- /^depends /{
- s/^depends //
- w '"${tmp_dir}"'/db.depends
- b
+ /^package-id /{
+ s/^\S\+ //
+ w '"${tmp_dir}"'/package-ids
+ d
}
'
+if [ ! -s "${tmp_dir}/packages" ]; then
+ printf 'Nothing to delete.\n'
+ exit
+fi
+
+mkdir "${tmp_dir}/repos"
+
+while read -r repo; do
+ ${master_mirror_rsync_command} \
+ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db.tar.gz" \
+ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files.tar.gz" \
+ "${tmp_dir}/repos/"
+
+ tar -Oxzf "${tmp_dir}/repos/${repo}.db.tar.gz" --wildcards '*/desc' | \
+ sed -n '
+ /^%FILENAME%$/{
+ N
+ s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/
+ T
+ p
+ }
+ ' | \
+ while read -r old new; do
+ printf 's,/%s\\(\\.sig\\)\\?$,/%s\\1,\n' \
+ "$(str_to_regex "${old}")" \
+ "$(str_to_regex "${new}")"
+ done >> \
+ "${tmp_dir}/sub_pkgrel-removal.sed"
+
+ # shellcheck disable=SC2046
+ repo-remove "${tmp_dir}/repos/${repo}.db.tar.gz" \
+ $(
+ grep "^$(str_to_regex "${repo}") " "${tmp_dir}/packages" | \
+ cut -d' ' -f2
+ )
+ if ! ${no_action}; then
+ ${master_mirror_rsync_command} \
+ "${tmp_dir}/repos/${repo}.db.tar.gz" \
+ "${tmp_dir}/repos/${repo}.files.tar.gz" \
+ "${master_mirror_rsync_directory}/i686/${repo}/"
+ fi
+done < \
+ "${tmp_dir}/repositories"
+
+if [ -s "${tmp_dir}/sub_pkgrel-removal.sed" ]; then
+ sed -i -f "${tmp_dir}/sub_pkgrel-removal.sed" "${tmp_dir}/package-files"
+fi
+if ${no_action}; then
+ printf 'Now, I would remove the packages from the database and delete the following files from the master mirror:\n'
+ sed '
+ s,^, ,
+ ' "${tmp_dir}/package-files"
+ exit
+fi
-echo 'OK' >&2
-exit 42
-
-# # sanity check
-#
-# for ending in 'done' 'testing'; do
-# if [ "${ending}" = 'testing' ] && \
-# [ -z "${packages_to_stabilize}" ]; then
-# # if nothing is to be untested, we don't care about duplicate
-# # testing packages (and maybe an unstaging fixes this anyway)
-# continue
-# fi
-# if [ -n "$(
-# find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
-# sed 's|\(\.[^.]\+\)\{4\}$||' | \
-# sort | \
-# uniq -d
-# )" ]; then
-# >&2 echo 'Removing duplicates not yet implemented:'
-# find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
-# sed 's|\(\.[^.]\+\)\{4\}$||' | \
-# sort | \
-# uniq -d
-# exit 42
-# fi
-# done
-#
-# # packages which are done
-#
-# find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
-# sed '
-# s|\.done$||
-# ' | \
-# sort -u > \
-# "${tmp_dir}/done-packages"
-#
-# # packages still on the build-list
-#
-# tr ' ' '.' < \
-# "${work_dir}/build-list" | \
-# sort -u > \
-# "${tmp_dir}/keep-packages"
-#
-# find "${work_dir}/package-infos" -name '*.groups' \
-# -exec grep -qx 'base\(-devel\)\?' {} \; \
-# -printf '%f\n' | \
-# sed '
-# s|\.groups$||
-# ' | \
-# sort -u > \
-# "${tmp_dir}/base-packages"
-#
-# # no base / base-devel packages on the build list?
-# if [ -z "$(
-# join -j 1 \
-# "${tmp_dir}/base-packages" \
-# "${tmp_dir}/keep-packages"
-# )" ]; then
-# # unstage all base / base-devel packages from staging
-# cat "${tmp_dir}/base-packages" "${tmp_dir}/base-packages" "${tmp_dir}/keep-packages" | \
-# sort | \
-# uniq -u | \
-# sponge "${tmp_dir}/keep-packages"
-# fi
-#
-# # find all dependencies of the unstageable packages
-# mv \
-# "${tmp_dir}/keep-packages" \
-# "${tmp_dir}/new-keep-packages"
-# touch "${tmp_dir}/keep-packages"
-#
-# while [ -s "${tmp_dir}/new-keep-packages" ]; do
-#
-# cat "${tmp_dir}/new-keep-packages" "${tmp_dir}/keep-packages" | \
-# sort -u | \
-# sponge "${tmp_dir}/keep-packages"
-#
-# sed '
-# s|^|'"${work_dir}"'/package-infos/|
-# s|$|.depends|
-# ' "${tmp_dir}/keep-packages" | \
-# xargs -r grep -HF '' | \
-# sed '
-# s|^.*/||
-# s|\.depends:| |
-# ' | \
-# sort -u | \
-# sort -k2,2 > \
-# "${tmp_dir}/keep-packages.depends"
-#
-# sed '
-# s|^|'"${work_dir}"'/package-infos/|
-# s|$|.builds|
-# ' "${tmp_dir}/done-packages" | \
-# xargs -r grep -HF '' | \
-# sed '
-# s|^.*/||
-# s|\.builds:| |
-# ' | \
-# sort -u | \
-# sort -k2,2 > \
-# "${tmp_dir}/done-packages.builds"
-#
-# join -j 2 -o 1.1 \
-# "${tmp_dir}/done-packages.builds" \
-# "${tmp_dir}/keep-packages.depends" | \
-# sort -u > \
-# "${tmp_dir}/new-keep-packages"
-#
-# # "new" is only what has not been there before
-# cat "${tmp_dir}/keep-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/new-keep-packages" | \
-# sort | \
-# uniq -u | \
-# sponge "${tmp_dir}/new-keep-packages"
-#
-# done
-#
-# # unlock build list
-#
-# rm -f "${build_list_lock_file}"
-# flock -u 9
-#
-# clean_up_lock_file() {
-# rm -rf --one-file-system "${tmp_dir}"
-# rm -f "${package_database_lock_file}"
-# }
-#
-# # calculate unstageable packages from keep_packages and done_packages
-#
-# done_packages=$(
-# cat \
-# "${tmp_dir}/keep-packages" \
-# "${tmp_dir}/keep-packages" \
-# "${tmp_dir}/done-packages" | \
-# sort | \
-# uniq -u
-# )
-#
-# # move packages in packages_to_stabilize from *testing/ to the stable repos
-#
-# # shellcheck disable=SC2046
-# move_packages $(
-#
-# for package in ${packages_to_stabilize}; do
-#
-# if [ -z "${package}" ]; then
-# continue
-# fi
-#
-# printf '%s/%s/%s\n' \
-# "${package}" \
-# "$(official_or_community "${package}" 'testing')" \
-# "$(repository_of_package "${package}")"
-#
-# done
-#
-# )
-#
-# # move packages from *staging to *testing
-#
-# # shellcheck disable=SC2046
-# move_packages $(
-#
-# for package in ${done_packages}; do
-#
-# if [ -z "${package}" ]; then
-# continue
-# fi
-#
-# printf '%s/%s/%s\n' \
-# "${package}" \
-# "$(official_or_community "${package}" 'staging')" \
-# "$(official_or_community "${package}" 'testing')"
-#
-# done
-#
-# )
-#
-# clean_up_lock_file
+# shellcheck disable=SC2016
+{
+ printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `to_deletes`;\n' \
+ "${tmp_dir}/package-ids"
+
+ printf 'UPDATE `binary_packages` '
+ printf ' JOIN `to_deletes` ON `to_deletes`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_architectures
+ printf ' SET `repository`=('
+ printf 'SELECT `repositories`.`id`'
+ printf ' FROM `repositories`'
+ printf ' WHERE `repositories`.`name`="deletion-list"'
+ printf ');\n'
+} | \
+ mysql_run_query
+
+sed '
+ s,^,rm "i686/,
+ s,$,",
+' "${tmp_dir}/package-files" | \
+ ${master_mirror_sftp_command}
diff --git a/bin/get-assignment b/bin/get-assignment
index d256248..7ac749f 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -15,49 +15,37 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
-mkdir -p "${work_dir}/package-states"
+# TODO: honor manual build order of tool-chain:
+# toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc
hand_out_assignment() {
- # find out the sub_pkgrel
- sub_pkgrel=$(
- next_sub_pkgrel "$1" "$2" "$3" "$4"
- )
-
- # we don't care anymore if an older version of this package was
- # "locked" or "broken"
- find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
- -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \
- -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \
- -delete
-
- # move that build order to the end of the build-list
- sed -i '
- /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
- $ b
- d
- }
- $ a '"$1 $2 $3 $4" \
- "${work_dir}/build-list"
-
- echo "$1 $2 $3 $4 ${sub_pkgrel}"
- {
- # shellcheck disable=SC2154
- echo "${slave}"
- if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
- cat "${work_dir}/package-states/$1.$2.$3.$4.locked"
- fi
- } | \
- sort -u | \
- sponge "${work_dir}/package-states/$1.$2.$3.$4.locked"
# shellcheck disable=SC2016
{
+ printf 'SELECT '
+ printf '`package_sources`.`%s`,' \
+ 'pkgbase' 'git_revision' 'mod_git_revision'
+ printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`'
+ printf ' FROM `upstream_repositories`'
+ mysql_join_upstream_repositories_package_sources
+ mysql_join_package_sources_build_assignments
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
+ 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
+ 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+ printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+ "$(printf '%s' "$4" | base64 -w0)"
+ printf ' LIMIT 1;\n'
+
printf 'UPDATE `build_slaves`'
printf ' SET `currently_building` = ('
printf ' SELECT `build_assignments`.`id`'
printf ' FROM `build_assignments`'
- printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
- printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
printf ' WHERE'
printf ' `package_sources`.`%s` = from_base64("%s") AND' \
'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
@@ -66,16 +54,24 @@ hand_out_assignment() {
printf ' `upstream_repositories`.`name` = from_base64("%s")' \
"$(printf '%s' "$4" | base64 -w0)"
printf ')'
- printf ' WHERE `build_slaves`.`name`=from_base64("%s");' \
+ # shellcheck disable=SC2154
+ printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
"$(printf '%s' "${slave}" | base64 -w0)"
- } | \
- ${mysql_command}
- # lock every loop this package breaks
- find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -exec grep -qxF "$1" '{}' \; \
- -exec touch '{}.locked' \;
+ printf 'UPDATE `build_assignments`'
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ printf ' SET `build_assignments`.`priority`=0'
+ printf ' WHERE'
+ printf ' `package_sources`.`%s` = from_base64("%s") AND' \
+ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
+ 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
+ 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+ printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \
+ "$(printf '%s' "$4" | base64 -w0)"
+ } | \
+ mysql_run_query | \
+ tr '\t' ' '
exit 0
@@ -108,140 +104,106 @@ clean_up() {
tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
trap clean_up EXIT
-# Check if there are any pending packages at all and if the requester
-# has already got an assignment.
-
-pending_packages=false
-
-while read -r package git_revision mod_git_revision repository; do
+# if we're building something already, hand it out (again)
+currently_building=$(
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT '
+ mysql_query_select_pkgbase_and_revision
+ mysql_join_build_assignments_build_slaves
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+ "$(printf '%s' "${slave}" | base64 -w0)"
+ printf ' AND `repositories`.`name`="build-list"'
+ printf ' LIMIT 1;\n'
+ } | \
+ mysql_run_query
+)
- generate_package_metadata "${package}.${git_revision}.${mod_git_revision}.${repository}"
+if [ -n "${currently_building}" ]; then
+ # shellcheck disable=SC2086
+ hand_out_assignment ${currently_building}
+fi
- if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then
- # has this slave already got or does he prefer this assignment?
- # note, that the dependencies are met, because this package is already locked
- if grep -qxF "${slave}" "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" || \
- printf ',%s,' "$1" | \
- grep -qF ",${package},"; then
- hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
- fi
- elif [ ! -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
- pending_packages=true
- fi
+# a package with [all dependencies met or which is part of a loop]
+# and which is currently not being built, ordered by:
+# 1: we requested it
+# 2: its priority
+# 3: is not yet built
+# 4: was built the longest time ago
+next_building=$(
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT '
+ printf '`package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \
+ "$(
+ printf '%s' "$1" | \
+ base64 -w0
+ )"
+ printf '`build_assignments`.`priority`,'
+ printf 'COALESCE('
+ printf 'MAX(`failed_builds`.`date`),0'
+ printf ') AS `last_trial`,'
+ mysql_query_is_part_of_loop '`build_assignments`.`id`'
+ printf ' AS `part_of_loop`,'
+ mysql_query_select_pkgbase_and_revision
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' LEFT'
+ mysql_join_build_assignments_failed_builds
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND NOT EXISTS ('
+ printf ' SELECT *'
+ printf ' FROM `build_slaves`'
+ printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`'
+ printf ') AND ('
+ printf '`build_assignments`.`is_blocked` IS NULL'
+ printf ' OR'
+ printf ' `package_sources`.`pkgbase`=from_base64("%s")' \
+ "$(
+ printf '%s' "$1" | \
+ base64 -w0
+ )"
+ printf ') AND ('
+ mysql_query_is_part_of_loop '`build_assignments`.`id`'
+ printf ' OR NOT '
+ mysql_query_has_pending_dependencies '`build_assignments`.`id`'
+ printf ')'
+ printf ' GROUP BY `build_assignments`.`id`'
+ printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`'
+ printf ' LIMIT 1;\n'
+ } | \
+ mysql_run_query | \
+ sed '
+ y/\t/ /
+ s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/
+ '
+)
+if [ -n "${next_building}" ]; then
+ # shellcheck disable=SC2086
+ hand_out_assignment ${next_building}
+fi
-done < "${work_dir}/build-list"
+# Check if there are any pending packages at all
+count_pending=$(
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT count(*)'
+ printf ' FROM `build_assignments`'
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND `build_assignments`.`is_blocked` IS NULL'
+ printf ';\n'
+ } | \
+ mysql_run_query
+)
-if ! ${pending_packages}; then
+if [ "${count_pending}" -eq 0 ]; then
>&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages'
exit 3
+else
+ >&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready'
+ exit 2
fi
-
-# Find first package of build-list whose "dependencies" are all met
-# 1st: prefered packages on the build list which have all dependencies met
-# 2nd: unbroken packages on the build list which have all dependencies met
-# 3rd: unbroken packages breaking a loop
-# 4th: broken packages which have all dependencies met or break a loop
-
-for iteration in 'prefered' 'fresh' 'loops' 'broken'; do
-
- case "${iteration}" in
- 'prefered')
- hand_out_blocked=true
- hand_out_broken=true
- hand_out_loop=false
- echo "$1" | \
- tr ',' '\n' | \
- sort -u > \
- "${tmp_dir}/hand-out-only-these-packages"
- ;;
- 'fresh')
- hand_out_blocked=false
- hand_out_broken=false
- hand_out_loop=false
- {
- cat "${work_dir}/build-list"
- find "${work_dir}/package-states" -name '*.broken' -printf '%f\n' | \
- sed '
- s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
- p
- '
- } | \
- sort -k1,1 -k2 | \
- uniq -u | \
- cut -d' ' -f1 | \
- uniq > \
- "${tmp_dir}/hand-out-only-these-packages"
- ;;
- 'loops')
- hand_out_blocked=false
- hand_out_broken=false
- hand_out_loop=true
- find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -exec cat {} \; | \
- sort -u > \
- "${tmp_dir}/hand-out-only-these-packages"
- ;;
- 'broken')
- hand_out_blocked=false
- hand_out_broken=true
- hand_out_loop=true
- cut -d' ' -f1 < \
- "${work_dir}/build-list" | \
- sort -u > \
- "${tmp_dir}/hand-out-only-these-packages"
- ;;
- esac
-
- cat -n "${work_dir}/build-list" | \
- sort -k2,2 | \
- join -1 1 -2 2 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/hand-out-only-these-packages" - | \
- sort -k1,1 | \
- sed 's|^\s*\S\+\s\+||' > \
- "${tmp_dir}/try-to-hand-out-these-packages"
-
- while read -r package git_revision mod_git_revision repository; do
-
- # package locked? (we have taken care of those in the previous loop)
- if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then
- continue
- fi
-
- # package blocked?
- if ! ${hand_out_blocked} &&
- [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
- continue
- fi
-
- # package broken?
- if ! ${hand_out_broken} && \
- [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then
- continue
- fi
-
- # dependencies met?
- if [ -n "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
- # do we hand out looped packages?
- if ! ${hand_out_loop}; then
- continue
- fi
- # is it a looped package?
- if ! find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -exec cat {} \; | \
- grep -qxF "${package}"; then
- continue
- fi
- fi
-
- hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
- done < \
- "${tmp_dir}/try-to-hand-out-these-packages"
-
-done
-
-# Remove the lock file
-
->&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready'
-exit 2
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 801aadd..cf2ea6c 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -1,6 +1,6 @@
#!/bin/sh
-# check for packages that need to be built, and build a list in the proper build order
+# check for packages that need to be built
# Details:
# https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates
@@ -9,9 +9,12 @@
# TODO: Find out, why sometimes package updates are missed.
-# TODO: mark loops in mysql database
+# TODO: read information from database
-# TODO: test_exclusion does not yet cooperate with the database
+# TODO: correctly handle if pkgbase of a split package is renamed, e.g.:
+# $a -> ($a,$b) ==> $b -> ($a,$b)
+
+# TODO: keep database clean in case of abort
# shellcheck disable=SC2016
usage() {
@@ -86,57 +89,68 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
exit
fi
+# TODO: How should new deletion-list packages be handled?
+# - packages deleted upstream should be marked as to-be-deleted if
+# existent and otherwise be ignored
+# - packages deleted due to black listing should be marked as black
+# listed - and thus as to-be-deleted, but they should remain in the
+# database after deletion!
+
# delete_package package
# mark $package for deletion
delete_package() {
- echo "$1" >> \
- "${work_dir}/deletion-list.new"
- sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
- # shellcheck disable=SC2016
- {
- printf 'UPDATE `binary_packages`'
- printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \
- 'repositories' 'repositories' 'repository' \
- 'build_assignments' 'build_assignments' 'build_assignment'
- printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
- 'package_sources' 'package_sources' 'build_assignments' 'package_source'
- printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
- printf ' WHERE `repositories`.`name`="build-list"'
- printf ' AND `package_sources`.`pkgbase`=from_base64("%s");' \
- "$(printf '%s' "$1" | base64 -w0)"
- } | \
- ${mysql_command}
+ # TODO: Once we want to rely on the database for test_exclusion, we
+ # need to run the command below unconditionally, but with some
+ # changes, so we can easily revert.
+ if [ -z "${test_exclusion}" ]; then
+ # shellcheck disable=SC2016
+ {
+ # packages from the build-list/to-be-decided go straight to the deletion-list
+ printf 'UPDATE `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")'
+ printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")'
+ printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \
+ "$(printf '%s' "$1" | base64 -w0)"
+ # other packages are marked as `is_to_be_deleted`
+ printf 'UPDATE `binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ printf ' SET `binary_packages`.`is_to_be_deleted`=1'
+ printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \
+ "$(printf '%s' "$1" | base64 -w0)"
+ } | \
+ mysql_run_query
+ fi
}
-# create tmp_dir and trap
-tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-# Update git repositories (official packages, community packages and the repository of package customizations).
+something_new=false
for repo in ${repo_names}; do
eval repo_path='"${repo_paths__'"${repo}"'}"'
+ # Update git repositories (official packages, community packages and the repository of package customizations).
git -C "${repo_path}" fetch origin master:master
-done
-
-# Read previous git revision numbers from files.
-
-something_new=false
-
-for repo in ${repo_names}; do
+ # read previous git revision numbers from database.
+ # shellcheck disable=SC2016
eval "old_repo_revisions__${repo}='$(
- cat "${work_dir}/${repo}.revision" 2> /dev/null || \
- echo NONE
+ {
+ printf 'SELECT `git_repositories`.`head` FROM `git_repositories`'
+ printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
+ "$(printf '%s' "${repo}" | base64 -w0)"
+ } | \
+ mysql_run_query
)'"
- eval repo_path='"${repo_paths__'"${repo}"'}"'
- eval "new_repo_revisions__${repo}='$(
- if ${pull}; then
+ # determine new git revision
+ if ${pull}; then
+ eval "new_repo_revisions__${repo}='$(
git -C "${repo_path}" rev-parse HEAD
- else
- cat "${work_dir}/${repo}.revision"
- fi | \
- tee "${work_dir}/${repo}.revision.new"
- )'"
+ )'"
+ else
+ eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"'
+ fi
if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then
something_new=true
fi
@@ -162,6 +176,11 @@ if ! flock -s ${block_flag} 8; then
exit
fi
+trap mysql_cleanup EXIT
+
+# shellcheck disable=SC2119
+mysql_cleanup
+
echo 'Check modified packages from the last update, and put them to the build list.'
# Check modified packages from the last update, and put them to the build list.
@@ -169,13 +188,6 @@ echo 'Check modified packages from the last update, and put them to the build li
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.
-cp \
- "${work_dir}/build-list" \
- "${work_dir}/build-list.new"
-cp \
- "${work_dir}/deletion-list" \
- "${work_dir}/deletion-list.new"
-
for repo in ${repo_names}; do
eval repo_path='"${repo_paths__'"${repo}"'}"'
eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
@@ -191,26 +203,9 @@ for repo in ${repo_names}; do
git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}"
fi
} | \
- # Packages which are already on the build list should receive a git_revision bump if _any_ file changed.
- # Thus, we rename any file "PKGBUILD" to trigger the successive logic.
- if [ "${repo}" = 'archlinux32' ]; then
- sed "$(
- sed '
- s/ .*$//
- s|^|\\@^.\\t[^/]\\+/|
- s|$|/@ s@/[^/]*$@/PKGBUILD@|
- ' "${work_dir}/build-list"
- )"
- else
- sed "$(
- sed '
- s/ .*$//
- s|^|\\@^.\\t|
- s|$|/@ s@/[^/]*$@/PKGBUILD@|
- ' "${work_dir}/build-list"
- )"
- fi | \
# only track changes in PKGBUILDs
+ # TODO: However, packages which are already on the build list should
+ # receive a git_revision bump if _any_ file changed.
grep '/PKGBUILD$' | \
if [ "${repo}" = "archlinux32" ]; then
# modify the directory structure from the modifiaction-repository
@@ -245,24 +240,26 @@ done | \
while read -r mode package git_revision repository; do
if [ "${mode}" = 'D' ]; then
# deleted PKGBUILD
- git_revision=$(cat "${work_dir}/archlinux32.revision.new")
+ # shellcheck disable=SC2154
+ git_revision="${new_repo_revisions__archlinux32}"
found_package=false
for repository in ${repo_names}; do
eval 'repo_path="${repo_paths__'"${repository}"'}"'
if [ "${repository}" = "archlinux32" ]; then
- if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" 2> /dev/null | \
+ if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \
tar -t 2> /dev/null | \
grep -q "/$(str_to_regex "${package}")/PKGBUILD$"; then
found_package=true
break;
fi
else
- if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" -- "${package}/repos" 2> /dev/null | \
- tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \
- cut -d/ -f3 | \
- grep -v 'staging\|testing\|-unstable' | \
- grep -vq -- '-i686$'; then
- git_revision=$(cat "${work_dir}/${repository}.revision.new")
+ # shellcheck disable=SC2154
+ if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos" 2> /dev/null | ' \
+ 'tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | ' \
+ 'cut -d/ -f3 | ' \
+ 'grep -v '"'"'staging\|testing\|-unstable'"'"' | ' \
+ 'grep -vq -- '"'"'-i686$'"'"; then
+ eval 'git_revision="${new_repo_revisions__'"${repository}"'}"'
found_package=true
break;
fi
@@ -280,31 +277,36 @@ done | \
fi
fi
if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
- # new or modified PKGBUILD
- sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
- # shellcheck disable=SC2154
- echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
- "${work_dir}/build-list.new"
- sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new"
# shellcheck disable=SC2016
{
# delete old build assignment and associated binary packages
- # which are not yet build or on the deletion list
+ # which are not yet built or on the deletion list
printf 'DELETE `build_assignments`,`binary_packages`'
printf ' FROM `binary_packages`'
- printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
- 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
- 'package_sources' 'package_sources' 'build_assignments' 'package_source' \
- 'repositories' 'repositories' 'binary_packages' 'repository'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_binary_packages_repositories
printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
"$(
printf '%s' "${package}" | \
base64 -w0
)"
printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n'
+ # remove is-to-be-deleted marker from old binary packages
+ printf 'UPDATE `binary_packages`'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_binary_packages_repositories
+ printf ' SET `is_to_be_deleted`=0'
+ printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \
+ "$(
+ printf '%s' "${package}" | \
+ base64 -w0
+ )"
} | \
- ${mysql_command}
- mysql_generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
+ mysql_run_query
+ # shellcheck disable=SC2154
+ mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
continue
fi
@@ -312,21 +314,6 @@ done | \
exit 1
done
-sort -u "${work_dir}/deletion-list.new" | \
- sponge "${work_dir}/deletion-list.new"
-
-echo 'Extract dependencies of packages.'
-
-# First, we extract the dependencies of each package.
-
-mkdir -p "${work_dir}/package-infos"
-
-while read -r package git_revision mod_git_revision repository; do
-
- generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
-done < "${work_dir}/build-list.new"
-
echo 'apply blacklisting'
# ignore blacklisted packages and dependent packages
# this is the first time when all the information is available and up to date
@@ -334,249 +321,190 @@ echo 'apply blacklisting'
black_listed=''
black_listed_new=$(
{
- git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \
- tar -Ox 'blacklist' | \
- sed '
- s/\s*#.*$//
- /^\s*$/d
- '
- if [ -n "${test_exclusion}" ]; then
- echo "${test_exclusion}"
- fi
- find "${work_dir}/package-infos" -maxdepth 1 -name 'lib32-*' -printf '%f\n' | \
- sed 's|\(\.[^.]\+\)\{4\}$||' | \
- uniq
+ {
+ git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
+ tar -Ox 'blacklist' | \
+ sed '
+ s/\s*#.*$//
+ /^\s*$/d
+ '
+ if [ -n "${test_exclusion}" ]; then
+ echo "${test_exclusion}"
+ fi
+ } | \
+ base64_encode_each
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")'
+ printf ' FROM `package_sources`'
+ printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
+ } | \
+ mysql_run_query
} | \
sort -u
)
-find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' -printf '%f\n' | \
- grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.builds$' | \
- sed '
- s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$|\2 \3 \4 \1|
- ' | \
- sort -u | \
- sort -k4,4 > \
- "${tmp_dir}/packages"
-
-{
- uniq -f3 -u "${tmp_dir}/packages" | \
- awk '{print $4 " " $4 "." $1 "." $2 "." $3}'
- uniq -f3 -D "${tmp_dir}/packages" | \
- uniq -f3 --group=append | \
- while read -r rev mod_rev repo pkg; do
- if [ -z "${rev}" ] && \
- [ -z "${mod_rev}" ] && \
- [ -z "${repo}" ] && \
- [ -z "${pkg}" ]; then
- # shellcheck disable=SC2031
- printf '%s %s.%s.%s.%s\n' \
- "${opkg}" \
- "${opkg}" \
- "$(
- # shellcheck disable=SC2086
- printf '%s\n' ${revs} | \
- sort -u | \
- find_newest_of_git_revisions
- )" \
- "$(
- # shellcheck disable=SC2086
- printf '%s\n' ${mod_revs} | \
- sort -u | \
- find_newest_of_git_revisions
- )" \
- "${orepo}" | \
- grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+$'
- revs=''
- mod_revs=''
- continue
- fi
- revs=$(
- printf '%s\n' \
- ${revs} \
- "${rev}"
- )
- mod_revs=$(
- printf '%s\n' \
- ${mod_revs} \
- "${mod_rev}"
- )
- opkg="${pkg}"
- orepo="${repo}"
- done
-} | \
- sort -k1,1 > \
- "${tmp_dir}/newest-revisions"
-
-sed '
- s|^\S\+ |'"${work_dir}"'/package-infos/|
- s|$|.builds|
-' "${tmp_dir}/newest-revisions" | \
- xargs -r cat | \
- sort | \
- uniq -c > \
- "${work_dir}/built-packages"
-
-sed '
- s|^\S\+ |'"${work_dir}"'/package-infos/|
- s|$|.build-depends|
-' "${tmp_dir}/newest-revisions" | \
- xargs -r grep -HF '' | \
- sed '
- s|\(\.[^.]\+\)\{4\}:\([^:]\+\)$| \2|
- s|^.*/||
- ' | \
- sort -k2,2 > \
- "${work_dir}/newest-dependencies"
-
while [ -n "${black_listed_new}" ]; do
black_listed=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${black_listed} ${black_listed_new} | \
+ printf '%s\n' "${black_listed}" "${black_listed_new}" | \
+ grep -vxF '' | \
sort -u
)
black_listed_new=$(
{
- # shellcheck disable=SC2086
- printf '%s\n' ${black_listed} | \
- sort -k1,1 | \
- join -j1 -o 2.2 - "${tmp_dir}/newest-revisions" | \
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.builds|
- ' | \
- xargs -r cat | \
- sort | \
- uniq -c
- cat "${work_dir}/built-packages"
+ printf '%s\n' "${black_listed}" "${black_listed}"
+
+ # shellcheck disable=SC2016
+ {
+ printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
+ printf 'INSERT INTO `bl` (`pkgbase`) VALUES '
+ printf '%s\n' "${black_listed}" | \
+ sort -u | \
+ sed '
+ s/^/(from_base64("/
+ s/$/")),/
+ $ s/,$/;/
+ '
+ printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")'
+ printf ' FROM `package_sources` AS `a_ps`'
+ mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
+ mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
+ mysql_join_binary_packages_dependencies 'a_bp'
+ printf ' WHERE NOT EXISTS ('
+ printf ' SELECT *'
+ printf ' FROM `install_target_providers`'
+ printf ' WHERE NOT EXISTS ('
+ printf ' SELECT *'
+ printf ' FROM `bl`'
+ printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
+ mysql_join_package_sources_build_assignments 'b_ps' 'b_ba'
+ mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp'
+ printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
+ printf ')'
+ printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ') AND EXISTS ('
+ # TODO: This should be corrected at the root: automatic install targets, which are bogus should
+ # not be added in the first place - but how do we detect that?
+ printf ' SELECT *'
+ printf ' FROM `install_target_providers`'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ');\n'
+ } | \
+ mysql_run_query | \
+ sort -u
} | \
- sort | \
- uniq -d | \
- awk '{print $2}' | \
- sort -k1,1 | \
- join -1 1 -2 2 -o 2.1 - "${work_dir}/newest-dependencies" | \
- sort -u
- )
- black_listed_new=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${black_listed} ${black_listed} ${black_listed_new} | \
+ grep -vxF '' | \
sort | \
uniq -u
)
done
-echo "${black_listed}" | \
- while read -r package; do
- [ -n "${package}" ] && \
- delete_package "${package}"
- done
-sort -u "${work_dir}/deletion-list.new" | \
- sponge "${work_dir}/deletion-list.new"
-
-
-if [ -n "${test_exclusion}" ]; then
- # we should not actually update the build-list et. al, but solely print this difference:
- if diff --color -u "${work_dir}/deletion-list" "${work_dir}/deletion-list.new"; then
- printf 'If you put "%s" on the blacklist, no additional packages will end up on the deletion list.\n' "${test_exclusion}"
- fi
- exit 0
-fi
-
-# Now we create the partial order.
-
-while read -r package git_revision mod_git_revision repository; do
- # add "$pkgname -> $build-target" to build-order list
- sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.builds"
- # add "$dependency -> $pkgname" to build-order list
- sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
- # add "base/base-devel -> $pkgname" to build-order list
- printf '%s '"${package}"'\n' 'base' 'base-devel'
-done \
- < "${work_dir}/build-list.new" \
- > "${work_dir}/build-order"
-
-if grep -vq '^\S\+ \S\+$' "${work_dir}/build-order"; then
- >&2 echo 'Created partial order file "build-order" is invalid.'
- exit 2
-fi
-
-echo 'Now actually sort it.'
-
-{
- # this part will have the correct build order, but all the infos are missing
- tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \
- nl -ba | \
- awk '{print $1 " not-git also-not-git whatever " $2}'
- # this part has all the infos, but possibly the wrong order
- awk '{print "0 " $2 " " $3 " " $4 " " $1}' "${work_dir}/build-list.new"
-} | \
- sort -k5,5 -k1nr | \
- # now, we have the correct order and the infos, but in adjacent lines
- uniq -f4 -D | \
- sed '/^0 /d;N;s|\n| |' | \
- # now in one line, each
- sort -k1n,1 | \
- awk '{print $5 " " $7 " " $8 " " $9}' > \
- "${work_dir}/build-list.new.new"
-
-rm --one-file-system -rf "${work_dir}/build-list.loops.new"
-mkdir "${work_dir}/build-list.loops.new"
-
-if [ -s "${work_dir}/tsort.error" ]; then
- >&2 echo 'WARNING: There is a dependency cycle!'
- >&2 cat "${work_dir}/tsort.error"
- >&2 echo
- >&2 echo 'I will continue anyway.'
- # save loops in separate files each, so breaking them is easier
- awk '
- /^tsort: \S+: input contains a loop:$/{
- n++;
- getline
- }
- {
- print $2 >"'"${work_dir}"'/build-list.loops.new/loop_" n
- }
- ' "${work_dir}/tsort.error"
-
- # remove lines from loop files which are no packages
- find "${work_dir}/build-list.loops.new" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -printf '%p\n' | \
- while read -r loop; do
- {
- sort -u "${loop}"
- cut -d' ' -f1 "${work_dir}/build-list.new.new" | \
- sort -u
- } | \
- sort | \
- uniq -d | \
- sponge "${loop}"
- done
-else
- rm "${work_dir}/tsort.error"
-fi
+black_listed=$(
+ printf '%s\n' "${black_listed}" | \
+ while read -r line; do
+ printf '%s' \
+ "${line}" | \
+ base64 -d
+ printf '\n'
+ done
+)
deletion_list_count=$(
- wc -l < \
- "${work_dir}/deletion-list.new"
+ # shellcheck disable=SC2086
+ printf '%s\n' ${black_listed} | \
+ wc -l
)
if [ "${deletion_list_count}" -gt 1000 ]; then
>&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
"${deletion_list_count}"
+ if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
+ {
+ printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
+ for repo_name in ${repo_names}; do
+ eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"'
+ eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"'
+ # shellcheck disable=SC2154
+ if [ "${old_revision}" != "${new_revision}" ]; then
+ printf '%s: %s -> %s\n' \
+ "${repo_name}" \
+ "${old_revision}" \
+ "${new_revision}"
+ fi
+ done
+ } | \
+ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
+ irc_say
+ fi
exit 3
fi
+rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
-# Move the .new-files to the actual files
+echo "${black_listed}" | \
+ while read -r package; do
+ if [ -z "${package}" ]; then
+ continue
+ fi
+ delete_package "${package}"
+ done
+
+
+if [ -n "${test_exclusion}" ]; then
+ # TODO: reimplement test_exclusion with information from the database
+ >&2 echo 'sry, not yet done.'
+ exit 0
+fi
+
+echo 'Done - mark decisions as final.'
-rm -rf --one-file-system "${work_dir}/build-list.loops"
+# shellcheck disable=SC2016
{
- printf '%s\n' "build-list.loops" "build-list.new" "build-list" "deletion-list"
- # shellcheck disable=SC2086
- printf '%s.revision\n' ${repo_names}
-} | \
- while read -r file; do
- mv "${work_dir}/${file}.new" "${work_dir}/${file}"
+ # save blacklist into database
+ printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64), `reason` TEXT);\n'
+ git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
+ tar -Ox 'blacklist' | \
+ sed -n '
+ s/^\(\S\+\)\s*#\s*/\1 /
+ T
+ p
+ ' | \
+ while read -r pkgbase reason; do
+ printf '(from_base64("%s"),from_base64("%s")),\n' \
+ "$(printf '%s' "${pkgbase}" | base64 -w0)" \
+ "$(printf '%s' "${reason}" | base64 -w0)"
+ done | \
+ sed '
+ 1 i INSERT INTO `bl` (`pkgbase`,`reason`) VALUES
+ $ s/,$/;/
+ '
+ printf 'UPDATE `build_assignments`'
+ mysql_join_build_assignments_package_sources
+ printf ' LEFT JOIN `bl` ON `package_sources`.`pkgbase`=`bl`.`pkgbase`'
+ printf ' SET `build_assignments`.`is_black_listed`=`bl`.`reason`;\n'
+ printf 'DROP TEMPORARY TABLE `bl`;\n'
+
+ # update hashes of repositories in mysql database
+ for repo in ${repo_names}; do
+ printf 'UPDATE `git_repositories`'
+ printf ' SET `git_repositories`.`head`=from_base64("%s")' \
+ "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}" | base64 -w0')"
+ printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
+ "$(printf '%s' "${repo}" | base64 -w0)"
done
+ # move binary_packages from "to-be-decided" to "build-list"
+ printf 'UPDATE `binary_packages`'
+ mysql_join_binary_packages_repositories '' 'from_repo'
+ printf ' SET `repository`=('
+ printf 'SELECT `to_repo`.`id`'
+ printf ' FROM `repositories` AS `to_repo`'
+ printf ' WHERE `to_repo`.`name`="build-list"'
+ printf ')'
+ printf ' WHERE `from_repo`.`name`="to-be-decided";\n'
+} | \
+ mysql_run_query
+
+# update loop list in database (beware, the packages are expected to be in "build-list",
+# not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list".
+mysql_find_build_assignment_loops
# Remove the lock file
diff --git a/bin/ii-connect b/bin/ii-connect
index 54e1eb0..cf332c0 100755
--- a/bin/ii-connect
+++ b/bin/ii-connect
@@ -8,7 +8,7 @@
# start ii if it is not running
if ! pgrep -x ii > /dev/null; then
rm -rf --one-file-system "${irc_dir}"
- screen -S ii -d -m ii -f buildmaster -n buildmaster
+ screen -S ii -d -m ii -s irc.freenode.net -f buildmaster -n buildmaster
sleep 10
fi
@@ -19,12 +19,12 @@ if tail -n1 "${irc_dir}/nickserv/out" 2> /dev/null | \
sponge "${irc_dir}/nickserv/in"
fi
-# join #archlinux-ports if not yet done
+# join #archlinux32 if not yet done
if ! grep ' buildmaster\.archlinux32\.org .* buildmaster$' "${irc_dir}/out" | \
tail -n1 | \
- grep -q ' #archlinux-ports '; then
+ grep -q ' #archlinux32 '; then
{
- echo '/j #archlinux-ports'
+ echo '/j #archlinux32'
echo '/WHO buildmaster'
} | \
sponge "${irc_dir}/in"
@@ -39,7 +39,6 @@ fi
if [ "$1" = 'watch' ]; then
while pgrep -x 'ii' > /dev/null; do
find "${irc_dir}" \
- -regextype sed \
-type f \
-name 'out' \
-printf '%p\n' | \
@@ -51,10 +50,10 @@ if [ "$1" = 'watch' ]; then
else
prefix=''
fi
- regex='^\S\+ \S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) '
+ regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) '
if grep -q "${regex}" "${out_file}"; then
sed -n '
- s/'"${regex}"'/\1 /
+ s/'"${regex}"'/\2 /
T
p
' "${out_file}" | \
diff --git a/bin/interpret-mail b/bin/interpret-mail
index d16c07f..83c8999 100755
--- a/bin/interpret-mail
+++ b/bin/interpret-mail
@@ -5,6 +5,8 @@
# TODO: enable email interface to delete packages
+# TODO: read information from database (?)
+
if [ $# -ne 0 ]; then
>&2 echo ''
>&2 echo 'usage: interpret-mail'
@@ -21,15 +23,18 @@ if [ $# -ne 0 ]; then
>&2 echo ' - "block: <state-file> <reason>":'
>&2 echo ' Block the given packge for the given reason.'
>&2 echo ''
+ >&2 echo ' - "copy-to-build-support: <pkgname>":'
+ >&2 echo ' Copy the given binary package into [build-support].'
+ >&2 echo ''
+ >&2 echo ' - "schedule: <pkgbase>":'
+ >&2 echo ' Put the given package on the build list (again).'
+ >&2 echo ''
>&2 echo ' - "stabilize: <package-file>":'
>&2 echo ' Mark the given package as tested.'
>&2 echo ''
>&2 echo ' - "unblock: <state-file>":'
>&2 echo ' Unblock the given packge.'
>&2 echo ''
- >&2 echo ' - "schedule: <pkgname>":'
- >&2 echo ' Put the given package on the build list (again).'
- >&2 echo ''
>&2 echo ' - ALL: all of the above (only valid in'
>&2 echo ' "conf/admin-gpg-keys")'
>&2 echo ''
@@ -156,14 +161,14 @@ sed -n '
/\n$/!ba
s/\n$//
p
-' "${tmp_dir}/plain-content" |
+' "${tmp_dir}/plain-content" | \
sed '
:start_loop
$!{
N
bstart_loop
}
- s/=\s*\n//g
+ s/[=\]\s*\n//g
s/:\s*\n/: /g
s/\n\(\S\+[^: ]\(\s\|\n\|$\)\)/ \1/g
' > \
@@ -185,7 +190,7 @@ sed -n "$(
)" "${tmp_dir}/raw-content"
if [ -s "${tmp_dir}/block" ]; then
- if run_and_log_on_error "${base_dir}/bin/modify-package-state" --block "${tmp_dir}/block"; then
+ if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --block "${tmp_dir}/block"; then
log 'Successfully blocked %s packages.\n' "$(wc -l < "${tmp_dir}/block")"
else
log 'There was an error while blocking the packages - ignoring this message.\n'
@@ -196,7 +201,7 @@ if [ -s "${tmp_dir}/stabilize" ]; then
sed -i '
/\.pkg\.tar\.xz$/!s/$/.pkg.tar.xz/
' "${tmp_dir}/stabilize"
- if run_and_log_on_error "${base_dir}/bin/modify-package-state" --tested "${tmp_dir}/stabilize"; then
+ if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --tested "${tmp_dir}/stabilize"; then
log 'Successfully marked %s packages as tested.\n' "$(wc -l < "${tmp_dir}/stabilize")"
else
log 'There was an error while marking the packages as tested - ignoring this message.\n'
@@ -204,7 +209,7 @@ if [ -s "${tmp_dir}/stabilize" ]; then
fi
if [ -s "${tmp_dir}/unblock" ]; then
- if run_and_log_on_error "${base_dir}/bin/modify-package-state" --unblock "${tmp_dir}/unblock"; then
+ if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --unblock "${tmp_dir}/unblock"; then
log 'Successfully unblocked %s packages.\n' "$(wc -l < "${tmp_dir}/unblock")"
else
log 'There was an error while unblocking the packages - ignoring this message.\n'
@@ -213,7 +218,7 @@ fi
if [ -s "${tmp_dir}/schedule" ]; then
# shellcheck disable=SC2046
- "${base_dir}/bin/seed-build-list" $(
+ "${base_dir}/bin/seed-build-list" --wait $(
tr '[:space:]' '\n' < \
"${tmp_dir}/schedule" | \
grep -vxF '' | \
@@ -224,3 +229,14 @@ if [ -s "${tmp_dir}/schedule" ]; then
sponge "${tmp_dir}/schedule"
log 'Successfully (re)scheduled %s packages.\n' "$(wc -l < "${tmp_dir}/schedule")"
fi
+
+if [ -s "${tmp_dir}/copy-to-build-support" ]; then
+ sed -i '
+ /\.pkg\.tar\.xz$/!s/$/.pkg.tar.xz/
+ ' "${tmp_dir}/copy-to-build-support"
+ if run_and_log_on_error "${base_dir}/bin/copy-to-build-support" --wait "${tmp_dir}/copy-to-build-support"; then
+ log 'Successfully copied %s packages to [build-support].\n' "$(wc -l < "${tmp_dir}/copy-to-build-support")"
+ else
+ log 'There was an error while copying the packages to [build-support] - ignoring this message.\n'
+ fi
+fi
diff --git a/bin/modify-package-state b/bin/modify-package-state
index c740d3d..4a2f5ac 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -1,5 +1,7 @@
#!/bin/sh
+# shellcheck disable=SC2039
+
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
@@ -16,25 +18,28 @@ usage() {
>&2 echo ' -n|--no-report: Do not report what packages were modified.'
>&2 echo ' -t|--tested: Mark package(s) as tested.'
>&2 echo ' -u|--unblock: Unblock package(s).'
+ >&2 echo ' -w|--wait: Wait for lock if necessary.'
>&2 echo ''
>&2 echo 'Exactly one of -b|-t|-u is needed for actual operation.'
[ -z "$1" ] && exit 1 || exit "$1"
}
eval set -- "$(
- getopt -o bfhntu \
+ getopt -o bfhntuw \
--long block \
--long faulty \
--long help \
--long no-report \
--long tested \
--long unblock \
+ --long wait \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
action=''
report=true
+wait_for_lock='-n'
while true
do
@@ -73,6 +78,9 @@ do
fi
action='unblock'
;;
+ -w|--wait)
+ wait_for_lock=''
+ ;;
--)
shift
break
@@ -95,26 +103,19 @@ if [ $# -ne 1 ]; then
usage
fi
-clean_up() {
- rm -rf --one-file-system "${tmp_dir}"
-}
-tmp_dir=$(mktemp -d 'tmp.modify-package-state.XXXXXXXXXX' --tmpdir)
-trap clean_up EXIT
-
input_file="$1"
if ${report}; then
if ! [ -w "${input_file}" ]; then
>&2 printf \
- 'Cannot open file "%s" for writing.' \
+ 'Cannot open file "%s" for writing.\n' \
"${input_file}"
exit 2
fi
move_output() {
cat "${output_file}" > "${input_file}"
- clean_up
+ rm -f "${output_file}"
}
- output_file="${tmp_dir}/output-file"
- touch "${output_file}"
+ output_file=$(mktemp 'tmp.modify-package-state.XXXXXXXXXX')
trap 'move_output' EXIT
else
output_file='/dev/null'
@@ -122,91 +123,130 @@ fi
if ! [ -r "${input_file}" ]; then
>&2 printf \
- 'Cannot open file "%s" for reading.' \
+ 'Cannot open file "%s" for reading.\n' \
"${input_file}"
exit 2
fi
exec 9> "${sanity_check_lock_file}"
-if ! flock -s -n 9; then
+if ! flock -s ${wait_for_lock} 9; then
>&2 echo 'Cannot get sanity-check lock.'
exit
fi
-if [ "${action}" = 'tested' ]; then
- receive_buglist 'Testing' > \
- "${tmp_dir}/package-bug-titles"
+exec 8> "${package_database_lock_file}"
+if ! flock ${wait_for_lock} 8; then
+ >&2 echo 'Cannot get package-database lock.'
+ exit
fi
-{
- err=0
- while read -r package reason; do
- if echo "${package}" | \
- grep -q '\.pkg\.tar\.xz$'; then
- package=$(
- find "${work_dir}/package-states" -maxdepth 1 \( -name '*.tested' -o -name '*.testing' \) \
- -exec grep -qxF "${package}" '{}' \; \
- -printf '%f\n' | \
- sed 's|\.[^.]\+$||' | \
- sort -u
+while read -r package reason; do
+ case "${action}" in
+ 'faulty'|'tested')
+ # we expect a binary package identifier (pkgname-epoch:pkgver-pkgrel.sub_pkgrel-arch[".pkg.tar.xz"])
+ # and we will only operate on packages in "testing" repositories
+ # shellcheck disable=SC2016
+ combiner=$(
+ printf '`binary_packages`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ mysql_join_binary_packages_architectures
)
- if [ -z "${package}" ]; then
- continue
- fi
- fi
- if [ "${action}" = 'block' ] || \
- [ "${action}" = 'unblock' ]; then
- # these packages need to be on the build list
- if ! tr ' ' '.' < \
- "${work_dir}/build-list" | \
- grep -qxF "${package}"; then
- >&2 printf 'Package "%s" is not on the build-list.\n' "${package}"
- err=2
- continue
+ # shellcheck disable=SC2016,SC2031
+ selector=$(
+ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name "${package}"
+ printf 'WHERE `repository_stabilities`.`name`="testing"'
+ printf ' AND `binary_packages`.`%s`=from_base64("%s")' \
+ 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \
+ 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
+ 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
+ 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
+ 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)"
+ printf ' AND `architectures`.`name`=from_base64("%s")' \
+ "$(printf '%s' "${arch}" | base64 -w0)"
+ )
+ ;;
+ 'block'|'unblock')
+ # we expect a package source identifier (pkgbase.git_revision.mod_git_revision.repository)
+ # and we will only operate on packages in "unbuilt" repositories
+ # shellcheck disable=SC2016
+ combiner=$(
+ printf '`build_assignments`'
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ )
+ # shellcheck disable=SC2016
+ selector=$(
+ repository="${package##*.}"
+ pkgbase="${package%.*}"
+ mod_git_revision="${pkgbase##*.}"
+ pkgbase="${pkgbase%.*}"
+ git_revision="${pkgbase##*.}"
+ pkgbase="${pkgbase%.*}"
+ printf 'WHERE `repository_stabilities`.`name`="unbuilt"'
+ printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+ "$(printf '%s' "${repository}" | base64 -w0)"
+ printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+ 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \
+ 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \
+ 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)"
+ )
+ ;;
+ *)
+ >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
+ exit 42
+ ;;
+ esac
+ case "${action}" in
+ 'block')
+ if [ -z "${reason}" ]; then
+ >&2 printf 'No reason is given for blocking package "%s".\n' "${package}"
+ exit 2
fi
- fi
- case "${action}" in
- 'block')
- if [ -z "${reason}" ]; then
- >&2 printf 'No reason is given for blocking package "%s".\n' "${package}"
- err=2
- else
- echo "${reason}" > \
- "${work_dir}/package-states/${package}.blocked"
- printf '%s %s\n' "${package}" "${reason}"
- fi
- ;;
- 'faulty')
- if [ -f "${work_dir}/package-states/${package}.tested" ]; then
- mv \
- "${work_dir}/package-states/${package}.tested" \
- "${work_dir}/package-states/${package}.testing"
- printf '%s\n' "${package}"
- fi
- ;;
- 'tested')
- if [ -f "${work_dir}/package-states/${package}.testing" ] && \
- ! grep -qF "[${package%.*.*.*}]" "${tmp_dir}/package-bug-titles"; then
- mv \
- "${work_dir}/package-states/${package}.testing" \
- "${work_dir}/package-states/${package}.tested"
- printf '%s\n' "${package}"
- fi
- ;;
- 'unblock')
- if [ -f "${work_dir}/package-states/${package}.blocked" ]; then
- rm "${work_dir}/package-states/${package}.blocked"
- printf '%s\n' "${package}"
- fi
- ;;
- *)
- >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
- exit 42
- ;;
- esac
- done > \
- "${output_file}"
-
- exit ${err}
-} < \
- "${input_file}"
+ tester='1'
+ # shellcheck disable=SC2016
+ modifier=$(
+ printf '`build_assignments`.`is_blocked`=from_base64("%s")' \
+ "$(printf '%s' "${reason}" | base64 -w0)"
+ )
+ ;;
+ 'unblock')
+ # shellcheck disable=SC2016
+ tester='NOT `build_assignments`.`is_blocked` IS NULL'
+ # shellcheck disable=SC2016
+ modifier='`build_assignments`.`is_blocked`=NULL'
+ ;;
+ 'faulty')
+ # shellcheck disable=SC2016
+ tester='`binary_packages`.`has_issues`=0'
+ # shellcheck disable=SC2016
+ modifier='`binary_packages`.`is_tested`=0,`binary_packages`.`has_issues`=1'
+ ;;
+ 'tested')
+ # shellcheck disable=SC2016
+ tester='`binary_packages`.`is_tested`=0'
+ # shellcheck disable=SC2016
+ modifier='`binary_packages`.`is_tested`=1'
+ ;;
+ *)
+ >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
+ exit 42
+ ;;
+ esac
+ if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \
+ mysql_run_query | \
+ grep -qxF '1'; then
+ # shellcheck disable=SC2016
+ {
+ printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}"
+ printf ';\n'
+ } | \
+ mysql_run_query
+ printf '%s %s\n' "${action}" "${package}"
+ fi
+done \
+ < "${input_file}" \
+ > "${output_file}"
diff --git a/bin/mysql-functions b/bin/mysql-functions
deleted file mode 100755
index ec2bf4c..0000000
--- a/bin/mysql-functions
+++ /dev/null
@@ -1,647 +0,0 @@
-#!/bin/sh
-
-# contains functions used to access mysql db
-
-# shellcheck disable=SC2039
-
-if [ -z "${base_dir}" ]; then
- # just to make shellcheck happy
- . 'conf/default.conf'
-fi
-
-# base64_encode_each encode each line of stdin with base64
-
-base64_encode_each() {
- local line
-
- while read -r line; do
- printf '%s' \
- "${line}" | \
- base64 -w0
- printf '\n'
- done
-}
-
-# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository
-
-# shellcheck disable=SC2016,SC2086
-mysql_add_package_source() {
- local names='pkgbase git_revision mod_git_revision upstream_package_repository'
- local values;
- for _ in ${names}; do
- values="${values}$(
- printf '%s' "$1" | \
- base64 -w0
- ) "
- shift
- done
- values="${values% }"
-
- {
- printf 'INSERT IGNORE INTO package_sources'
- printf ' ('
- printf '`%s`, ' ${names}
- printf ') SELECT'
- printf ' from_base64("%s"), ' ${values% *}
- printf ' `upstream_repositories`.`id`'
- printf ' FROM `upstream_repositories`'
- printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \
- "${values##* }"
- } | \
- sed 's|, )|)|g' | \
- ${mysql_command}
-}
-
-# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository
-
-# shellcheck disable=SC2016,SC2031,SC2086,SC2154
-mysql_add_binary_package() {
- local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository'
- local name
- for name in ${names}; do
- eval 'local '"${name}"
- eval "${name}"'=$(
- printf "%s" "$1" |
- base64 -w0
- )'
- shift
- done
-
- {
- printf 'INSERT IGNORE INTO binary_packages'
- printf ' ('
- printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture'
- printf ') SELECT'
- printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}"
- printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures'
- printf ' FROM'
- printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures'
- printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
- printf ' WHERE'
- printf ' `%s`.`name` = from_base64("%s") AND' \
- 'repositories' "${repository}" \
- 'architectures' "${architecture}"
- printf ' `package_sources`.`%s` = from_base64("%s") AND' \
- 'pkgbase' "${pkgbase}" \
- 'git_revision' "${git_revision}" \
- 'mod_git_revision' "${mod_git_revision}"
- printf ' `upstream_repositories`.`name` = from_base64("%s")' \
- "${upstream_package_repository}"
- } | \
- sed '
- s|, )|)|g
- s|, FROM| FROM|g
- ' | \
- ${mysql_command}
-}
-
-# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel
-
-# shellcheck disable=SC2016,SC2031,SC2086,SC2154
-mysql_show_binary_package() {
- local names='pkgname pkgver pkgrel sub_pkgrel'
- local name
- for name in ${names}; do
- eval 'local '"${name}"
- eval "${name}"'=$(
- printf "%s" "$1" |
- base64 -w0
- )'
- shift
- done
-
- local joint
- {
- printf 'SELECT'
- printf ' `%s`.`%s`,' \
- 'repositories' 'name' \
- 'binary_packages' 'pkgname' \
- 'package_sources' 'pkgver' \
- 'package_sources' 'pkgrel' \
- 'binary_packages' 'sub_pkgrel' \
- 'architectures' 'name' \
- 'package_sources' 'pkgbase' \
- 'package_sources' 'git_revision' \
- 'package_sources' 'mod_git_revision' \
- 'upstream_repositories' 'name'
- printf ' FROM `binary_packages`'
- for joint in \
- 'architectures:binary_packages:architecture' \
- 'package_sources:binary_packages:package_source' \
- 'repositories:binary_packages:repository' \
- 'upstream_repositories:package_sources:upstream_package_repository'; do
- printf ' JOIN `%s` ON `%s`.`id` =' \
- "${joint%%:*}" "${joint%%:*}"
- joint="${joint#*:}"
- printf ' `%s`.`%s`' \
- "${joint%:*}" "${joint#*:}"
- done
- printf ' WHERE'
- printf ' `%s`.`%s` = from_base64("%s") AND' \
- 'binary_packages' 'pkgname' "${pkgname}" \
- 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \
- 'package_sources' 'pkgver' "${pkgver}" \
- 'package_sources' 'pkgrel' "${pkgrel}"
- printf ';'
- } | \
- sed '
- s|, FROM| FROM|g
- s|AND;|;|g
- ' | \
- ${mysql_command} --html
-}
-
-# mysql_add_install_target $install_target
-
-# shellcheck disable=SC2016,2086
-mysql_add_install_target() {
- local install_target
- install_target=$(
- printf "%s" "$1" | \
- base64 -w0
- )
-
- {
- printf 'INSERT IGNORE INTO `install_targets` (`name`)'
- printf ' VALUES (from_base64("%s"))' \
- "${install_target}"
- } | \
- ${mysql_command}
-}
-
-# mysql_generate_package_metadata $package $git_revision $mod_git_revision $repository
-# or
-# mysql_generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# if sub_pkgrel should be determined automatically and the package is on the build-list
-# and
-# mysql_generate_package_metadata $sub_pkgrel $package $current_repository $git_revision $mod_git_revision $repository
-# or
-# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository
-# if $sub_pkgrel should be forced and the package is currently in $current_repository
-# generate the meta data of a package (dependencies, built packages, ...) in the database
-
-# shellcheck disable=SC2016
-mysql_generate_package_metadata() {
-
- ( # new shell is intentional
- case "$1" in
- ''|*[!0-9]*)
- unset forced_sub_pkgrel
- current_repository=$(
- printf 'build-list' | \
- base64 -w0
- )
- ;;
- *)
- forced_sub_pkgrel=$(
- printf '%s' "$1" | \
- base64 -w0
- )
- shift
- current_repository=$(
- printf '%s' "$1" | \
- base64 -w0
- )
- shift
- ;;
- esac
- package="$1"
- git_revision="$2"
- mod_git_revision="$3"
- repository="$4"
- temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
- if [ $# -eq 1 ]; then
- # second form
- repository="${package##*.}"
- package="${package%.*}"
- mod_git_revision="${package##*.}"
- package="${package%.*}"
- git_revision="${package##*.}"
- package="${package%.*}"
- fi
-
- printf '.' >&2
- if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then
- printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"
- exit 2
- fi
- # remove empty lines and unsupported architectures
- sed -i '
- /^[^=]*=\s*$/d
- /^\s*arch = /{
- / \(i686\|any\)$/!d
- }
- ' "${temp_dir}/SRCINFO"
-
- if [ ! -s "${temp_dir}/SRCINFO" ]; then
- >&2 printf '"make_source_info" had empty output - eh, what?\n'
- exit 2
- fi
- printf '\n\n' >> "${temp_dir}/SRCINFO"
-
- printf '.' >&2
- pkgbase=$(
- grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \
- cut -d' ' -f3
- )
- if [ -z "${pkgbase}" ]; then
- >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n'
- exit 2
- fi
-
- # add the package source
- mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
- printf '.' >&2
-
- # now we encode everything in base64
- pkgbase=$(
- printf '%s' "${pkgbase}" | \
- base64 -w0
- )
- git_revision=$(
- printf '%s' "${git_revision}" | \
- base64 -w0
- )
- mod_git_revision=$(
- printf '%s' "${mod_git_revision}" | \
- base64 -w0
- )
- repository=$(
- printf '%s' "${repository}" | \
- base64 -w0
- )
-
- # add the build assignment(s)
- {
- archs=$(
- sed -n '
- s/^\tarch = //
- T
- p
- ' "${temp_dir}/SRCINFO" | \
- grep -vxF 'any' | \
- sort -u
- )
- if [ -z "${archs}" ]; then
- echo 'any'
- else
- printf '%s\n' "${archs}"
- fi
- } | \
- while read -r arch; do
- printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`)'
- printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL'
- printf ' FROM `architectures` JOIN `package_sources`'
- printf ' WHERE `architectures`.`name` = from_base64("%s")' \
- "$(
- printf '%s' "${arch}" | \
- base64 -w0
- )"
- printf ' AND `package_sources`.`%s` = from_base64("%s")' \
- 'pkgbase' "${pkgbase}" \
- 'git_revision' "${git_revision}" \
- 'mod_git_revision' "${mod_git_revision}"
- printf ';\n'
- done > \
- "${temp_dir}/add-build-assignments-command"
-
- # TODO: correctly link between binary_packages and build_assignments using any_arch
-
- # shellcheck disable=SC2034
- # select any specific arch (which will be building the 'any' part of a split package)
- any_arch=$(
- {
- sed -n '
- s/^\tarch = //
- T
- p
- ' "${temp_dir}/SRCINFO" | \
- sort -r | \
- grep -vxFm 1 'any' || \
- echo 'any'
- } | \
- base64_encode_each
- )
-
- grep '^pkgname = ' "${temp_dir}/SRCINFO" | \
- cut -d' ' -f3 | \
- while read -r pkgname; do
- pkgname64=$(
- printf '%s' "${pkgname}" | \
- base64 -w0
- )
- sed -n '
- /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{
- :a
- N
- /\n$/{
- p
- T
- }
- ba
- }
- ' "${temp_dir}/SRCINFO" | \
- sed '
- /^\S/d
- s/^\s*//
- ' > \
- "${temp_dir}/BINARYINFO.${pkgname64}"
-
- grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \
- cut -d' ' -f3 | \
- while read -r arch; do
- arch64=$(
- printf '%s' "${arch}" | \
- base64 -w0
- )
- sed '
- s/^\(\S\+\)_'"${arch}"' = /\1 = /
- ' "${temp_dir}/BINARYINFO.${pkgname64}" > \
- "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}"
- done
- done
- find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \
- while read -r _ pkgname arch; do
- pkgver=$(
- grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- base64_encode_each
- )
- pkgrel=$(
- grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- base64_encode_each
- )
- epoch=$(
- {
- grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \
- echo 'epoch = 0'
- } | \
- cut -d' ' -f3 | \
- base64_encode_each
- )
- provides=$(
- grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- builddepends=$(
- grep '^\(checkdepends\|makedepends\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- rundepends=$(
- grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- if [ -n "${forced_sub_pkgrel}" ]; then
- sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")'
- else
- sub_pkgrel=$(
- printf '(SELECT COALESCE('
- # do not add binary packages which are currently on the build-list
- printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
- printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
- printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
- printf ' WHERE'
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- printf ' `architectures`.`name`=from_base64("%s")' \
- "${arch}"
- printf ' AND `repositories`.`name`="build-list"),'
- # max(sub_pkgrel)+1
- printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
- printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
- printf ' WHERE'
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then
- # 'any' gets higher sub_pkgrel than any architecture
- printf ' 1'
- else
- # not-'any' gets higher sub_pkgrel than same or 'any' architecture
- printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \
- "${arch}"
- fi
- printf ')'
- printf ',0))'
- )
- fi
- {
- printf 'INSERT IGNORE INTO `binary_packages` ('
- printf '`%s`,' \
- 'build_assignment' \
- 'repository' \
- 'architecture' \
- 'epoch' \
- 'pkgver' \
- 'pkgrel' \
- 'pkgname' \
- 'sub_pkgrel' \
- 'has_issues' \
- 'is_tested'
- printf ') SELECT '
- printf '`%s`.`id`,' \
- 'build_assignments' \
- 'repositories' \
- 'architectures'
- printf 'from_base64("%s"),' \
- "${epoch}" \
- "${pkgver}" \
- "${pkgrel}" \
- "${pkgname}"
- printf '%s,0,0 FROM' \
- "${sub_pkgrel}"
- printf ' `%s` JOIN' \
- 'repositories' \
- 'architectures' \
- 'build_assignments' \
- 'package_sources'
- printf ' ON `build_assignments`.`package_source` = `package_sources`.`id`'
- printf ' JOIN `upstream_repositories`'
- printf ' ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`'
- printf ' WHERE'
- printf ' `%s`.`%s` = from_base64("%s") AND' \
- 'repositories' 'name' "${current_repository}" \
- 'architectures' 'name' "${arch}" \
- 'package_sources' 'pkgbase' "${pkgbase}" \
- 'package_sources' 'git_revision' "${git_revision}" \
- 'package_sources' 'mod_git_revision' "${mod_git_revision}" \
- 'upstream_repositories' 'name' "${repository}"
- printf ';\n'
- } | \
- sed '
- s|,)|)|g
- s|JOIN ON|ON|g
- s| AND;$|;|
- ' >> \
- "${temp_dir}/add-binary-packages-command"
- {
- printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \
- 'provides' \
- 'builddepends' \
- 'rundepends'
-
- printf 'INSERT INTO `provides` VALUES\n'
- echo "${provides}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' (from_base64("%s"));\n' \
- "${pkgname}"
-
- printf 'INSERT INTO `rundepends` VALUES\n'
- echo "${rundepends}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' ("base");\n'
-
- printf 'INSERT INTO `builddepends` VALUES\n'
- echo "${builddepends}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' ("base-devel");\n'
-
- printf 'INSERT IGNORE INTO `install_targets` (`name`)'
- printf ' SELECT (`name`) FROM `%s` UNION' \
- 'provides' \
- 'builddepends' \
- 'rundepends' | \
- sed 's| UNION$|;\n|'
-
- for link in 'provides' 'builddepends' 'rundepends'; do
- case "${link}" in
- 'provides')
- printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT'
- printf ' `binary_packages`.`id`,`install_targets`.`id` FROM'
- ;;
- 'builddepends'|'rundepends')
- printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT'
- printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM'
- printf ' `dependency_types` JOIN'
- ;;
- esac
- printf ' `binary_packages`'
- printf ' JOIN `architectures`'
- printf ' ON `binary_packages`.`architecture` = `architectures`.`id`'
- printf ' JOIN `repositories`'
- printf ' ON `binary_packages`.`repository` = `repositories`.`id`'
- printf ' JOIN `install_targets`'
- printf ' JOIN `%s`' "${link}"
- printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
- printf ' WHERE'
- if [ "${link}" = 'builddepends' ] || \
- [ "${link}" = 'rundepends' ]; then
- printf ' `dependency_types`.`name` = "%s" AND' \
- "${link%depends}"
- fi
- printf ' `binary_packages`.`%s` = from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- printf ' `binary_packages`.`sub_pkgrel` = %s AND' \
- "${sub_pkgrel}"
- printf ' `architectures`.`name` = from_base64("%s") AND' \
- "${arch}"
- printf ' `repositories`.`name` = from_base64("%s");\n' \
- "${current_repository}"
- done
-
- printf 'DROP TABLE `%s`;\n' \
- 'provides' \
- 'builddepends' \
- 'rundepends'
- } >> \
- "${temp_dir}/add-install-targets-command"
- done
- printf '.' >&2
-
- {
- if [ -s "${temp_dir}/add-build-assignments-command" ]; then
- cat "${temp_dir}/add-build-assignments-command"
- fi
- if [ -s "${temp_dir}/add-binary-packages-command" ]; then
- cat "${temp_dir}/add-binary-packages-command"
- fi
- if [ -s "${temp_dir}/add-install-targets-command" ]; then
- cat "${temp_dir}/add-install-targets-command"
- fi
- } | \
- ${mysql_command}
- printf '.' >&2
-
- )
-}
-
-# mysql_sanity_check
-# do a sanity check on the mysql database
-
-mysql_sanity_check() {
- ( # new shell is intentional
- temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
- for dir in $(ls_master_mirror 'i686'); do
- ls_master_mirror "i686/${dir}" | \
- sed -n '
- s/\.pkg\.tar\.xz$//
- T
- s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/
- s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/
- s|^|'"${dir}"'/|
- p
- '
- done | \
- sort > \
- "${temp_dir}/master-mirror-listing"
-
- # shellcheck disable=SC2016
- {
- printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`'
- printf ' FROM `binary_packages`'
- printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
- printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
- printf ' WHERE `repositories`.`is_on_master_mirror`'
- } | \
- ${mysql_command} --batch | \
- sed '
- 1d
- s,\t,/,
- s,\t,-,
- s,\t,:,
- s,\t,-,
- s,\t,.,
- s,\t,-,
- ' | \
- sort > \
- "${temp_dir}/mysql-packages"
-
- diff -u \
- "${temp_dir}/master-mirror-listing" \
- "${temp_dir}/mysql-packages"
- )
-}
diff --git a/bin/ping-from-slave b/bin/ping-from-slave
new file mode 100755
index 0000000..afaf018
--- /dev/null
+++ b/bin/ping-from-slave
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# should be called periodically on the build-master from the slaves to
+# - report any update on the build process
+# - show that the build is still running
+# - get notified by the build master if the build is not necessary anymore
+
+# shellcheck source=conf/default.conf
+. "${0%/*}/../conf/default.conf"
+
+# TODO: receive/save some statistics about current build
+
+if [ "$(
+ # shellcheck disable=SC2016,SC2154
+ {
+ printf 'SELECT count(*) FROM `build_slaves`'
+ mysql_join_build_slaves_build_assignments
+ printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
+ "$(printf '%s' "${slave}" | base64 -w0)"
+ } | \
+ mysql_run_query
+ )" -ne 1 ]; then
+ >&2 echo 'You do not build anything currently - abort whatever you are doing.'
+ exit 2
+fi
diff --git a/bin/ping-to-master b/bin/ping-to-master
new file mode 100755
index 0000000..15c9120
--- /dev/null
+++ b/bin/ping-to-master
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# periodically connects to the buildmaster to call ping-from slave to:
+# - report any update on the build process
+# - show that the build is still running
+# - get notified by the build master if the build is not necessary anymore
+
+# shellcheck source=conf/default.conf
+. "${0%/*}/../conf/default.conf"
+
+# TODO: abort build if requested to
+
+parent_pid="$1"
+parent_tmp_dir="$2"
+
+exec 9> "${work_dir}/ping-build-master.lock"
+
+while kill -0 "${parent_pid}" && \
+ [ -f "${parent_tmp_dir}/.ping-build-master" ]; do
+
+ # shellcheck disable=SC2029
+ find "${parent_tmp_dir}" \
+ -xdev \
+ -type f \
+ -name '*.build-log' \
+ -exec wc -l {} \; | \
+ sed 's, .*/, ,' | \
+ ssh \
+ -i "${master_build_server_identity}" \
+ -p "${master_build_server_port}" \
+ "${master_build_server_user}@${master_build_server}" \
+ 'ping-from-slave' "$(cat "${parent_tmp_dir}/.ping-build-master")"
+
+ # we wait upto 60 seconds to get the lock - if we get it, the parent
+ # must have released it and we're finished
+ if flock -w 60 9; then
+ break
+ fi
+done
diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list
index 06d4c06..db828ae 100755
--- a/bin/prioritize-build-list
+++ b/bin/prioritize-build-list
@@ -23,11 +23,24 @@ if ! flock -s -n 8; then
exit 1
fi
+# shellcheck disable=SC2016
{
- sed -n "/^$1/p" "${work_dir}/build-list"
- sed "/^$1/d" "${work_dir}/build-list"
+ printf 'UPDATE `build_assignments`'
+ mysql_join_build_assignments_package_sources
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' SET `build_assignments`.`priority`=('
+ printf 'SELECT COALESCE(MAX(`all_priorities`.`priority`),0)+1'
+ printf ' FROM ('
+ printf 'SELECT `others`.`priority`'
+ printf ' FROM `build_assignments` AS `others`'
+ printf ') AS `all_priorities`'
+ printf ')'
+ printf ' WHERE `package_sources`.`pkgbase` REGEXP from_base64("%s")' \
+ "$(printf '%s' "$1" | base64 -w0)"
+ printf ' AND `repositories`.`name`="build-list";\n'
} | \
- sponge "${work_dir}/build-list"
+ mysql_run_query
# Remove the lock file
diff --git a/bin/repo-copy b/bin/repo-copy
new file mode 100755
index 0000000..c3450fd
--- /dev/null
+++ b/bin/repo-copy
@@ -0,0 +1,92 @@
+#!/bin/sh
+
+# Rudimentary copy a package from one repository to another.
+# Note, that we do _not_ need to have the package itself, since all
+# relevant information is already in the original package database.
+
+# "Rudimentary" means the following restrictions:
+# - no arguments are accepted
+# - no database signatures are handled
+# - only *.db.tar.gz and *.files.tar.gz are recognized as database
+
+usage() {
+ >&2 echo 'usage:'
+ >&2 echo ' repo-copy from-repo.db.tar.gz to-repo.db.tar.gz package1 package2 ...'
+ >&2 echo
+ >&2 echo 'Note, that the packages must be given with version, e.g. "linux-4.15.7-1.0".'
+ exit 2
+}
+
+if [ $# -le 2 ]; then
+ usage
+fi
+
+from_repo="$1"
+to_repo="$2"
+shift
+shift
+
+tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/repo-tools.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+
+# extract the databases
+for repo in 'from' 'to'; do
+ for archive in 'db' 'files'; do
+ eval 'repo_db="${'"${repo}"'_repo}"'
+ if [ "${repo_db}" = "${repo_db%.db.tar.gz}" ]; then
+ >&2 printf '"%s" has an invalid suffix.\n' "${repo_db}"
+ usage
+ fi
+ if [ "${archive}" = 'files' ]; then
+ repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+ fi
+ if [ ! -f "${repo_db}" ]; then
+ >&2 printf 'Cannot open file "%s".\n' "${repo_db}"
+ usage
+ fi
+ mkdir "${tmp_dir}/${repo}.${archive}"
+ bsdtar -C "${tmp_dir}/${repo}.${archive}" -xf "${repo_db}"
+ done
+done
+
+# move the packages
+for package in "$@"; do
+ errors=$(
+ find "${tmp_dir}/to.db" "${tmp_dir}/to.files" -mindepth 1 -maxdepth 1 \
+ -printf '%f\n' | \
+ sed 's/-[^-]\+-[^-]\+$//' | \
+ grep -xF "${package%-*-*}"
+ )
+ if [ -n "${errors}" ]; then
+ >&2 printf 'The target repository "%s" already contains the following packages - "repo-remove" them first:\n' \
+ "${to_repo}"
+ >&2 printf '%s\n' "${errors}"
+ exit 2
+ fi
+ for archive in 'db' 'files'; do
+ if [ ! -d "${tmp_dir}/from.${archive}/${package}" ]; then
+ >&2 printf 'Repository "%s" does not contain package "%s"\n' \
+ "${from_repo}" "${package}"
+ exit 2
+ fi
+ mv "${tmp_dir}/from.${archive}/${package}" "${tmp_dir}/to.${archive}/"
+ done
+done
+
+# pack the database
+for archive in 'db' 'files'; do
+ repo_db="${to_repo}"
+ if [ "${archive}" = 'files' ]; then
+ repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+ fi
+ bsdtar -C "${tmp_dir}" -czf "${tmp_dir}/${repo_db##*/}" --strip-components=1 "to.${archive}"
+done
+
+# move the database in place
+for archive in 'db' 'files'; do
+ repo_db="${to_repo}"
+ if [ "${archive}" = 'files' ]; then
+ repo_db="${repo_db%.db.tar.gz}.files.tar.gz"
+ fi
+ mv "${tmp_dir}/${repo_db##*/}" "${repo_db}"
+done
diff --git a/bin/return-assignment b/bin/return-assignment
index cd6d30e..89a1f85 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -47,17 +47,37 @@ trap clean_up_lock_file EXIT
if [ "$5" = 'ERROR' ]; then
# the build failed on the build slave
- if ! grep -qxF "$1 $2 $3 $4" "${work_dir}/build-list" ||
- ! [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
- >&2 echo 'Too late, package already outdated, I ignore this error report.'
- exit 2
- fi
-
- # shellcheck disable=SC2154
- if ! grep -qxF "${slave}" "${work_dir}/package-states/$1.$2.$3.$4.locked"; then
- >&2 echo 'You do not build this package - move on.'
+ # shellcheck disable=SC2016
+ infos=$(
+ {
+ printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`'
+ mysql_join_build_slaves_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+ "$(
+ # shellcheck disable=SC2154
+ printf '%s' "${slave}" | \
+ base64 -w0
+ )"
+ printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
+ 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
+ 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+ printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+ "$(printf '%s' "$4" | base64 -w0)"
+ printf ' AND `repositories`.`name`="build-list"'
+ } | \
+ mysql_run_query | \
+ tr '\t' ' '
+ )
+ if [ -z "${infos}" ]; then
+ >&2 echo 'You do not build this package (anymore) - move on.'
exit 2
fi
+ was_broken_before="${infos##* }"
# save sent build logs
saved_build_logs=$(
@@ -69,15 +89,6 @@ if [ "$5" = 'ERROR' ]; then
'*.build-log.gz'
)
- if [ -f "${work_dir}/package-states/$1.$2.$3.$4.broken" ]; then
- was_broken_before=true
- else
- was_broken_before=false
- fi
-
- # shellcheck disable=SC2154
- echo "${slave}" >> \
- "${work_dir}/package-states/$1.$2.$3.$4.broken"
# shellcheck disable=SC2016
{
if [ -n "${saved_build_logs}" ]; then
@@ -94,8 +105,7 @@ if [ "$5" = 'ERROR' ]; then
printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")'
printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`'
} | \
- ${mysql_command} --raw --batch | \
- sed '1d'
+ mysql_run_query
)
for saved_build_log in ${saved_build_logs}; do
printf '%s' "${fail_reason_identifiers}" | \
@@ -139,7 +149,7 @@ if [ "$5" = 'ERROR' ]; then
printf 'DROP TABLE `failures`;\n'
fi
printf 'UPDATE `build_assignments`'
- printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
+ mysql_join_build_assignments_build_slaves
printf ' SET `build_assignments`.`is_broken`=1, `build_slaves`.`currently_building`=NULL'
printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
"$(
@@ -147,36 +157,7 @@ if [ "$5" = 'ERROR' ]; then
base64 -w0
)"
} | \
- ${mysql_command}
-
- # shellcheck disable=SC2154
- sed -i '
- /^'"$(str_to_regex "${slave}")"'$/d
- ' "${work_dir}/package-states/$1.$2.$3.$4.locked"
- if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
- rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
-
- # unlock every loop this package would have broken and which is not
- # broken by another locked package
- locked_packages=$(
- find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' -printf '%f\n' | \
- sed 's@^\(.\+\)\.\([0-9a-f]\{40\}\.\)\{2\}[^.]\+\.locked$@\1@'
- )
- find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -exec grep -qxF "$1" '{}' \; \
- -not -exec grep -qxF "${locked_packages}" '{}' \; \
- -exec rm '{}.locked' \;
-
- # move that build order to the end of the build-list
- sed -i '
- /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
- $ b
- d
- }
- $ a '"$1 $2 $3 $4" \
- "${work_dir}/build-list"
- fi
+ mysql_run_query
# release lock on build-list - otherwise seed-build-list won't run
flock -u 9
@@ -195,7 +176,7 @@ if [ "$5" = 'ERROR' ]; then
' | \
tr ' ' '\n' | \
sed '
- s/^/-p ^haskell-/
+ s/^/-p ^(haskell-)?/
s/-[0-9.]\+$/\$/
' | \
sort -u
@@ -209,21 +190,19 @@ if [ "$5" = 'ERROR' ]; then
fi
)
- if [ -p "${irc_dir}/#archlinux-ports/in" ]; then
- {
- printf '%s is broken (says %s).' \
- "$1" \
- "${slave}"
- if [ -n "${rescheduled_packages}" ]; then
- printf -- ' - I rescheduled:'
- # shellcheck disable=SC2086
- printf ' %s,' ${rescheduled_packages} | \
- sed 's/,$/./'
- fi
- printf '\n'
- } | \
- sponge "${irc_dir}/#archlinux-ports/in"
- fi
+ {
+ printf '%s is broken (says %s).' \
+ "$1" \
+ "${slave}"
+ if [ -n "${rescheduled_packages}" ]; then
+ printf -- ' - I rescheduled:'
+ # shellcheck disable=SC2086
+ printf ' %s,' ${rescheduled_packages} | \
+ sed 's/,$/./'
+ fi
+ printf '\n'
+ } | \
+ irc_say
fi
exit 0
@@ -245,19 +224,33 @@ clean_up_lock_file() {
rm -f "${package_database_lock_file}"
}
-if ! grep -qxF "$1 $2 $3 $4" "${work_dir}/build-list" ||
- ! [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ] ||
- ! [ "$5" = "$(next_sub_pkgrel "$1" "$2" "$3" "$4")" ]; then
+# shellcheck disable=SC2016
+if ! {
+ printf 'SELECT count(*)'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_build_slaves
+ mysql_join_binary_packages_repositories
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND `build_slaves`.`name`=from_base64("%s")' \
+ "$(printf '%s' "${slave}" | base64 -w0)"
+ printf ' AND `package_sources`.`%s`=from_base64("%s")' \
+ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
+ 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
+ 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
+ printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
+ "$(printf '%s' "$4" | base64 -w0)"
+ printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \
+ "$(printf '%s' "$5" | base64 -w0)"
+ } | \
+ mysql_run_query | \
+ grep -qvxF '0'; then
>&2 echo 'Sorry, the sent package is outdated.'
exit 2
fi
-# shellcheck disable=SC2154
-if ! grep -qxF "${slave}" "${work_dir}/package-states/$1.$2.$3.$4.locked"; then
- >&2 echo 'Whoops, this package is not built by this slave.'
- exit 2
-fi
-
clean_up_tmp_dir() {
cd "${base_dir}"
rm -rf --one-file-system "${tmp_dir}"
@@ -274,17 +267,41 @@ tar -x \
--no-wildcards-match-slash \
'*.pkg.tar.xz' \
'*.pkg.tar.xz.sig' \
- '*.pkg.tar.xz-namcap.log.gz'
-
-# check if all packages are signed and all signatures belong to a package
+ '*.pkg.tar.xz-namcap.log.gz' \
+ '*.pkg.tar.xz.so.needs.gz' \
+ '*.pkg.tar.xz.so.provides.gz'
+
+# check if all packages come with:
+# - a package file
+# - a signature
+# - a namcap log
+# - a list of needed libraries
+# - a list of provided libraries
missing_files=$(
- find . -maxdepth 1 -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' -o -name '*.pkg.tar.xz-namcap.log.gz' | \
+ find . -maxdepth 1 \( \
+ \( \
+ -name '*.pkg.tar.xz' \
+ -printf '%f package\n' \
+ \) -o \
+ \( \
+ -name '*.pkg.tar.xz.sig' \
+ -printf '%f signature\n' \
+ \) -o \
+ \( \
+ -name '*.pkg.tar.xz-namcap.log.gz' \
+ -printf '%f namcap\n' \
+ \) -o \
+ \( \
+ -name '*.pkg.tar.xz.so.needs.gz' \
+ -printf '%f needed-libraries\n' \
+ \) -o \
+ \( \
+ -name '*.pkg.tar.xz.so.provides.gz' \
+ -printf '%f provided-libraries\n' \
+ \) \
+ \) | \
sed '
- s@\.sig$@ signature@
- t
- s@-namcap\.log\.gz$@ namcap@
- t
- s@$@ package@
+ s/\(\.pkg\.tar\.xz\)\(\.sig\|-namcap\.log\.gz\|\.so\.\(provides\|needs\)\.gz\) /\1 /
' | \
sort -k1,1 -k2,2 | \
sed '
@@ -315,11 +332,23 @@ missing_files=$(
p
g
}
+ / needed-libraries /!{
+ h
+ s/^\(\S\+\) .*$/List of libraries needed by "\1" is missing./
+ p
+ g
+ }
+ / provided-libraries /!{
+ h
+ s/^\(\S\+\) .*$/List of libraries provided by "\1" is missing./
+ p
+ g
+ }
'
)
if [ -n "${missing_files}" ]; then
- >&2 echo 'The following packages lack a signature, namcap log or package file:'
+ >&2 echo 'The following packages lack a signature, namcap log, package file or list of needed/provided libraries:'
>&2 echo "${missing_files}"
exit 3
fi
@@ -353,16 +382,27 @@ package_errors=$(
# shellcheck disable=SC2086
printf '%s\n' ${packages} | \
sed '
- s@\(-[^-]\+\)\{2\}-\([^-]\+\)\.pkg\.tar\.xz$@ \2@
- / any$/{
- s|any$|i686|
- }
s|^|was_built: |
'
- sed '
- s|$| i686|
- s|^|expected: |
- ' "${work_dir}/package-infos/$1.$2.$3.$4.packages"
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT CONCAT('
+ printf '"expected: ",'
+ mysql_package_name_query
+ printf ')'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_architectures
+ mysql_join_binary_packages_build_slaves
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+ "$(
+ printf '%s' "${slave}" | \
+ base64 -w0
+ )"
+ printf ' AND `repositories`.`name`="build-list"'
+ printf ';\n'
+ } | \
+ mysql_run_query
} | \
sort -k2 | \
uniq -u -f1
@@ -377,6 +417,62 @@ fi
# move namcap.logs
find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_log_directory}/success/" \;
+# insert provided libraries into database
+# shellcheck disable=SC2016
+{
+ for lib_link in 'pl:provides' 'nl:needs'; do
+ printf 'CREATE TEMPORARY TABLE `%s` (`pkgfile` VARCHAR(64), `lib` VARCHAR(128));\n' \
+ "${lib_link%:*}"
+ find . -maxdepth 1 -name '*.pkg.tar.xz.so.'"${lib_link#*:}"'.gz' -execdir zgrep -HF '' '{}' \; | \
+ sed -n '
+ s,^\./\(.\+\.pkg\.tar\.xz\)\.so\.'"${lib_link#*:}"'\.gz:\([^:]\+\)$,\1\n\2,
+ T
+ p
+ ' | \
+ base64_encode_each | \
+ sed '
+ N
+ s/^\(\S\+\)\n\(\S\+\)$/(from_base64("\1"),from_base64("\2")),/
+ $s/,$/;/
+ 2 s/^/INSERT INTO `'"${lib_link%:*}"'` (`pkgfile`,`lib`) VALUES /
+ '
+ printf 'INSERT IGNORE INTO `install_targets` (`name`)'
+ printf ' SELECT DISTINCT `%s`.`lib` FROM `%s`;\n' \
+ "${lib_link%:*}" "${lib_link%:*}"
+ if [ "${lib_link%:*}" = 'pl' ]; then
+ printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`)'
+ else
+ printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`)'
+ fi
+ printf ' SELECT `binary_packages`.`id`,`install_targets`.`id`'
+ if [ "${lib_link%:*}" = 'nl' ]; then
+ printf ',`dependency_types`.`id`'
+ fi
+ printf ' FROM `install_targets`'
+ if [ "${lib_link%:*}" = 'nl' ]; then
+ printf ' JOIN `dependency_types` ON `dependency_types`.`name`="link"'
+ fi
+ printf ' JOIN `%s` ON `%s`.`lib`=`install_targets`.`name`' \
+ "${lib_link%:*}" "${lib_link%:*}"
+ printf ' JOIN `binary_packages`'
+ mysql_join_binary_packages_architectures
+ mysql_join_binary_packages_build_slaves
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
+ "$(
+ printf '%s' "${slave}" | \
+ base64 -w0
+ )"
+ printf ' AND `repositories`.`name`="build-list"'
+ printf ' AND '
+ mysql_package_name_query
+ printf '=`%s`.`pkgfile`' \
+ "${lib_link%:*}"
+ printf ';\n'
+ done
+} | \
+ mysql_run_query
+
# move packages
destination=$(official_or_community "$1.$2.$3.$4" 'staging')
@@ -401,58 +497,46 @@ for package in ${packages}; do
remove_old_package_versions 'i686' "${destination}" "${package}"
done
-# remove old state files (these should be only "done" markers, but
-# actually we don't care what it is) - as long as it's not "testing" or "tested"
-find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
- -not -name '*.testing' \
- -not -name '*.tested' \
- -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \
- -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \
- -exec rm '{}' \;
-
-# remove all loops which are broken by this package
-find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
- -regex '.*/loop_[0-9]\+' \
- -exec grep -qxF "$1" '{}' \; \
- -exec rm '{}.locked' \;
-
-# remove package from build list
-sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
-
-# remove package lock file
-if ! [ "${destination}" = 'build-support' ]; then
- # shellcheck disable=SC2086
- printf '%s\n' ${packages} > \
- "${work_dir}/package-states/$1.$2.$3.$4.done"
-fi
# shellcheck disable=SC2016
{
printf 'UPDATE `build_assignments`'
- printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`'
- printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+ mysql_join_build_assignments_build_slaves
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_repositories '' 'old_repo'
printf ' SET'
printf ' `build_assignments`.`is_broken`=0,'
- printf ' `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s")),' \
+ printf ' `build_assignments`.`priority`=0,'
+ printf ' `binary_packages`.`repository`=(SELECT `new_repo`.`id` FROM `repositories` AS `new_repo` WHERE `new_repo`.`name`=from_base64("%s")),' \
"$(
printf '%s' "${destination}" | \
base64 -w0
)"
printf ' `binary_packages`.`has_issues`=0,'
printf ' `binary_packages`.`is_tested`=0'
- printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
+ printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
"$(
printf '%s' "${slave}" | \
base64 -w0
)"
- printf 'UPDATE `build_slaves` SET'
- printf ' `build_slaves`.`currently_building`=NULL'
- printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
+ printf ' AND `old_repo`.`name`="build-list";\n'
+ printf 'UPDATE `build_slaves` AS `to_update`'
+ printf ' JOIN `build_slaves` AS `current_slave`'
+ printf ' ON `to_update`.`currently_building`=`current_slave`.`currently_building`'
+ printf ' SET `to_update`.`currently_building`=NULL'
+ printf ' WHERE `current_slave`.`name`=from_base64("%s");\n' \
"$(
printf '%s' "${slave}" | \
base64 -w0
)"
+ printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n'
+ printf 'INSERT INTO `loops_to_delete`'
+ printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`'
+ mysql_join_build_dependency_loops_binary_packages
+ mysql_join_binary_packages_repositories
+ printf ' WHERE NOT `repositories`.`name` = "build-list";\n'
+ printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS ('
+ printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`'
+ printf ');\n'
+ printf 'DROP TABLE `loops_to_delete`;\n'
} | \
- ${mysql_command}
-rm -f \
- "${work_dir}/package-states/$1.$2.$3.$4.locked" \
- "${work_dir}/package-states/$1.$2.$3.$4.broken"
+ mysql_run_query
diff --git a/bin/sanity-check b/bin/sanity-check
index 8736d47..7d1b2f0 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -20,7 +20,7 @@ usage() {
i_am_insane() {
if [ ! -s "${work_dir}/build-master-sanity" ]; then
printf '\001ACTION goes insane.\001\n' | \
- sponge "${irc_dir}/#archlinux-ports/in"
+ irc_say
fi
echo 'build master is insane' > \
"${work_dir}/build-master-sanity"
@@ -103,7 +103,7 @@ touch "${tmp_dir}/messages"
trap 'finish' EXIT
if [ $# -eq 0 ]; then
- set -- git-repositories build-list repos package-database state-files
+ set -- git-repositories build-list mysql repos package-database track-state
fi
while [ $# -gt 0 ]; do
@@ -119,16 +119,28 @@ while [ $# -gt 0 ]; do
for repo in ${repo_names}; do
eval 'repo_path="${repo_paths__'"${repo}"'}"'
repo_revision=$(
- cat "${work_dir}/${repo}.revision"
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT `git_repositories`.`head` FROM `git_repositories`'
+ printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
+ "$(printf '%s' "${repo}" | base64 -w0)"
+ } | \
+ mysql_run_query
)
- if ! git -C "${repo_path}" archive "${repo_revision}" -- | \
- tar -t > /dev/null; then
+ if ! obj_type=$(git -C "${repo_path}" cat-file -t "${repo_revision}" 2>/dev/null); then
if [ ${silence} -le 1 ]; then
- printf '\nThe repository %s (%s) cannot archive the current revision %s.\n' \
+ printf '\nThe repository %s (%s) does not know the current revision %s.\n' \
"${repo}" "${repo_path}" "${repo_revision}" >> \
"${tmp_dir}/messages"
fi
i_am_insane
+ elif [ "${obj_type}" != 'commit' ]; then
+ if [ ${silence} -le 1 ]; then
+ printf '\nThe repository %s (%s) knows the current revision %s, but it is not a commit, but a %s.\n' \
+ "${repo}" "${repo_path}" "${repo_revision}" "${obj_type}" >> \
+ "${tmp_dir}/messages"
+ fi
+ i_am_insane
fi
done
@@ -145,20 +157,19 @@ while [ $# -gt 0 ]; do
"${tmp_dir}/messages"
errors=$(
- grep -vn '^\S\+ [0-9a-f]\{40\} [0-9a-f]\{40\} \S\+$' "${work_dir}/build-list"
- ) || true
- if [ -n "${errors}" ]; then
- if [ ${silence} -le 1 ]; then
- printf '\nThe following build orders are wrongly formatted:\n%s\n' \
- "${errors}" >> \
- "${tmp_dir}/messages"
- fi
- i_am_insane
- fi
-
- errors=$(
- cut -d' ' -f1 < \
- "${work_dir}/build-list" | \
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT `architectures`.`name`,`package_sources`.`pkgbase` FROM `package_sources`'
+ mysql_join_package_sources_build_assignments
+ mysql_join_build_assignments_architectures
+ printf ' WHERE EXISTS('
+ printf 'SELECT * FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' WHERE `repositories`.`name`="build-list"'
+ printf ' AND `binary_packages`.`build_assignment`=`build_assignments`.`id`'
+ printf ');\n'
+ } | \
+ mysql_run_query | \
sort | \
uniq -d
)
@@ -172,13 +183,16 @@ while [ $# -gt 0 ]; do
fi
errors=$(
+ # shellcheck disable=SC2016
{
- cut -d' ' -f1 < \
- "${work_dir}/build-list"
- cat "${work_dir}/deletion-list"
+ printf 'SELECT `a`.`pkgname` FROM `binary_packages` AS `a`'
+ mysql_join_binary_packages_repositories 'a' 'a_r'
+ printf ' AND `a_r`.`name`="build-list"'
+ printf ' JOIN `binary_packages` AS `b` ON `a`.`pkgname`=`b`.`pkgname`'
+ mysql_join_binary_packages_repositories 'b' 'b_r'
+ printf ' AND `b_r`.`name`="deletion-list";\n'
} | \
- sort | \
- uniq -d
+ mysql_run_query
)
if [ -n "${errors}" ]; then
if [ ${silence} -le 1 ]; then
@@ -268,17 +282,18 @@ while [ $# -gt 0 ]; do
errors=$(
{
- tar -tzf "${tmp_dir}/${repo}.db.tar.gz" | \
- grep '/$' | \
- sed '
- s|/$||
- s|^|in_database |
+ tar -Oxzf "${tmp_dir}/${repo}.db.tar.gz" --wildcards '*/desc' 2>/dev/null | \
+ sed -n '
+ /^%FILENAME%$/ {
+ N
+ s/^.*\n/in_database /
+ p
+ }
'
echo "${packages}" | \
- grep '\S' | \
sed '
- s|-[^-]\+$||
- s|^|in_repository |
+ /\.pkg\.tar\.xz$/ !d
+ s/^/in_repository /
' | \
sort -u
} | \
@@ -334,51 +349,80 @@ while [ $# -gt 0 ]; do
;;
- state-files)
-
- for status in 'staging' 'testing'; do
+ track-state)
- [ ${silence} -gt 0 ] || \
- printf 'checking state-files of "%s" ...' "${status}" >> \
- "${tmp_dir}/messages"
+ [ ${silence} -gt 0 ] || \
+ printf 'checking if all packages are tracked correctly ...' >> \
+ "${tmp_dir}/messages"
- errors=$(
+ errors=$(
+ {
+ # shellcheck disable=SC2016
{
- if [ "${status}" = 'staging' ]; then
- find "${work_dir}/package-states" -name '*.done' \
- -exec sed 's|^|package-state-file |' '{}' \;
- else
- find "${work_dir}/package-states" \( -name '*.testing' -o -name '*.tested' \) \
- -exec sed 's|^|package-state-file |' '{}' \;
- fi
- ls_master_mirror 'i686' | \
- grep "${status}\$" | \
- while read -r repo; do
- ls_master_mirror "i686/${repo}"
- done | \
- grep '\.pkg\.tar\.xz$' | \
- sed 's|^|package-file |'
+ printf 'SELECT "mysql",CONCAT(`repositories`.`name`,"/",'
+ mysql_package_name_query
+ printf ') FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ mysql_join_binary_packages_architectures
} | \
- sort -k2 | \
- uniq -cf1 | \
- grep -v '^\s*2\s' | \
- awk '{print $2 " " $3}'
- )
- if [ -n "${errors}" ]; then
- if [ ${silence} -le 1 ]; then
- printf '\nThe following %s packages do not have state files or vice versa:\n%s\n' \
- "${status}" \
- "${errors}" >> \
- "${tmp_dir}/messages"
- fi
- i_am_insane
+ mysql_run_query | \
+ tr '\t' ' '
+ ls_master_mirror 'i686' | \
+ while read -r repo; do
+ ls_master_mirror "i686/${repo}" | \
+ sed '
+ /\.pkg\.tar\.xz$/!d
+ s,^,package-file '"${repo}"'/,
+ '
+ done
+ } | \
+ sed 's/\(-[0-9]\+\)\.0\(-[^- ]\+$\)/\1\2/' | \
+ sort -k2 | \
+ uniq -uf1
+ )
+ if [ -n "${errors}" ]; then
+ if [ ${silence} -le 1 ]; then
+ printf '\nThe following packages from the master mirror are not tracked in the database or vice versa:\n%s\n' \
+ "${errors}" >> \
+ "${tmp_dir}/messages"
fi
+ i_am_insane
+ fi
- [ ${silence} -gt 0 ] || \
- echo ' passed.' >> \
+ [ ${silence} -gt 0 ] || \
+ echo ' passed.' >> \
+ "${tmp_dir}/messages"
+
+ ;;
+
+ mysql)
+
+ [ ${silence} -gt 0 ] || \
+ printf 'checking mysql-sanity-check-file ...' >> \
+ "${tmp_dir}/messages"
+
+ if [ -s "${webserver_directory}/mysql-sanity.html" ]; then
+ if [ ${silence} -le 1 ]; then
+ printf '\nThere is something wrong with the database:\n'
+ cat "${webserver_directory}/mysql-sanity.html"
+ fi >> \
"${tmp_dir}/messages"
+ i_am_insane
+ fi
- done
+ # hopefully, this gets rid of false positives :-)
+ sleep 1
+
+ if find "${work_dir}" -mindepth 1 -maxdepth 1 -name 'tmp.mysql-functions.query.*' | \
+ grep '\S' >> \
+ "${tmp_dir}/messages"; then
+ i_am_insane
+ fi
+
+ [ ${silence} -gt 0 ] || \
+ echo ' passed.' >> \
+ "${tmp_dir}/messages"
;;
@@ -399,5 +443,5 @@ done
if [ -f "${work_dir}/build-master-sanity" ]; then
rm "${work_dir}/build-master-sanity"
printf '\001ACTION resumes sanity.\001\n' | \
- sponge "${irc_dir}/#archlinux-ports/in"
+ irc_say
fi
diff --git a/bin/seed-build-list b/bin/seed-build-list
index acf25c3..ae08d57 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -5,8 +5,6 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
-# TODO: put new packages also in mysql database
-
# shellcheck disable=SC2016
usage() {
>&2 echo ''
@@ -16,6 +14,9 @@ usage() {
>&2 echo ' - a list of packages which need to be rebuilt'
>&2 echo ''
>&2 echo 'possible options:'
+ >&2 echo ' -a|--auto:'
+ >&2 echo ' Automatically reschedule packages which have run-time'
+ >&2 echo ' dependencies that are not available anywhere.'
>&2 echo ' -h|--help:'
>&2 echo ' Show this help and exit.'
>&2 echo ' -i|--ignore $package:'
@@ -26,6 +27,8 @@ usage() {
>&2 echo ' Do not actually update build-list, just print it.'
>&2 echo ' -p|--package $pkg_regex:'
>&2 echo ' Update packages matching $pkg_regex.'
+ >&2 echo ' -w|--wait:'
+ >&2 echo ' Wait for lock if necessary.'
[ -z "$1" ] && exit 1 || exit "$1"
}
@@ -34,26 +37,32 @@ tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT
eval set -- "$(
- getopt -o hi:m:np: \
+ getopt -o ahi:m:np:w \
+ --long auto \
--long help \
--long ignore: \
--long mirror: \
--long no-action \
--long package: \
+ --long wait \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
touch "${tmp_dir}/mirrors"
-touch "${tmp_dir}/delta-packages"
touch "${tmp_dir}/package-regexes"
touch "${tmp_dir}/ignore-packages"
+auto=false
update=true
+wait_for_lock='-n'
while true
do
case "$1" in
+ -a|--auto)
+ auto=true
+ ;;
-h|--help)
usage 0
;;
@@ -75,6 +84,9 @@ do
echo "$1" >> \
"${tmp_dir}/package-regexes"
;;
+ -w|--wait)
+ wait_for_lock=''
+ ;;
--)
shift
break
@@ -92,207 +104,280 @@ if [ $# -ne 0 ]; then
fi
if [ ! -s "${tmp_dir}/mirrors" ] && \
- [ ! -s "${tmp_dir}/package-regexes" ]; then
+ [ ! -s "${tmp_dir}/package-regexes" ] && \
+ ! ${auto}; then
# nothing to do
exit 0
fi
-repos="${stable_package_repositories}"
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f\n' | \
- sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
- sort -k1,1 > \
- "${tmp_dir}/known-packages"
-
-mod_repo_rev=$(cat "${work_dir}/archlinux32.revision")
-{
- for repo in ${repo_names}; do
- if [ "${repo_names}" = 'archlinux32' ]; then
- continue
- fi
- eval 'repo_path="${repo_paths__'"${repo}"'}"'
- repo_rev=$(cat "${work_dir}/${repo}.revision")
- git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
- tar -t | \
- grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
- grep -v -- '-i686/PKGBUILD$' | \
- grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
- sed '
- s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|'"${repo_rev}"' \1 \2|
- '
- done | \
- sort -u | \
- sort -k1,1
- awk '{print "nothing " $1 " " $4}' "${tmp_dir}/known-packages" | \
- sort -u | \
- sed 'p'
-} | \
- sort -k2,3 | \
- uniq -uf1 | \
- while read -r repo_rev pkg prepo; do
- generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}"
- done
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
- sed '
- s|^.*/||
- s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 |
- ' | \
- sort -k5,5 > \
- "${tmp_dir}/known-packages"
+# get locks
+if ${update}; then
+ exec 9> "${sanity_check_lock_file}"
+ if ! flock -s ${wait_for_lock} 9; then
+ >&2 echo 'Cannot get sanity-check lock.'
+ exit 1
+ fi
-# generate delta-packages from package-regexes
-while read -r pkg_regex; do
- if [ -z "${pkg_regex}" ]; then
- continue
+ exec 8> "${build_list_lock_file}"
+ if ! flock ${wait_for_lock} 8; then
+ >&2 echo 'Cannot get build-list lock.'
+ exit 1
fi
- awk '{print $5}' "${tmp_dir}/known-packages" | \
- grep "${pkg_regex}" || \
- true
-done < \
- "${tmp_dir}/package-regexes" >> \
- "${tmp_dir}/delta-packages"
+fi
-# genereate delta_packages from mirror delta
+repos="${stable_package_repositories}"
-while read -r mirror; do
- if [ -z "${mirror}" ]; then
- continue
- fi
+# genereate must-haves query from mirror delta
+if [ -s "${tmp_dir}/mirrors" ]; then
{
# theirs
- for repo in ${repos}; do
- curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
- tar -tz
- done | \
- grep '/$' | \
- sed '
- s|/$||
- s|^\(.*\)-\([^-]\+-[^-]\+\)|theirs \1-\2 \2 \1|
- '
-
+ while read -r mirror; do
+ if [ -z "${mirror}" ]; then
+ continue
+ fi
+ for repo in ${repos}; do
+ curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
+ tar -Oxz --wildcards '*/desc' | \
+ sed '
+ /^%FILENAME%$/!d
+ N
+ s/^.*\n//
+ s/-x86_64\(\.pkg\.tar\.xz\)$/-i686\1/
+ s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/theirs \2 \3 \1/
+ '
+ done
+ done < \
+ "${tmp_dir}/mirrors"
# ours
- for repo in $(ls_master_mirror 'i686'); do
- ls_master_mirror "i686/${repo}" | \
- grep '\.pkg\.tar\.xz$' | \
- sed 's|-[^-]\+$||'
- done | \
+ # shellcheck disable=SC2016
+ {
+ printf 'SELECT '
+ mysql_package_name_query
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_architectures
+ } | \
+ mysql_run_query | \
sed '
- s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1|
+ s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/
'
} | \
- expand_version 3 | \
- sort -k4,4 -k3Vr,3 -k1,1 | \
- shrink_version 3 | \
- uniq -f3 | \
- grep '^theirs ' | \
- awk '{print $4}' | \
- sort -k1,1 >> \
- "${tmp_dir}/delta-packages"
-done < \
- "${tmp_dir}/mirrors"
-
-sort -k1,1 -u "${tmp_dir}/delta-packages" | \
- sponge "${tmp_dir}/delta-packages"
-
-cat \
- "${tmp_dir}/delta-packages" \
- "${tmp_dir}/ignore-packages" \
- "${tmp_dir}/ignore-packages" | \
- sort | \
- uniq -u | \
- sponge "${tmp_dir}/delta-packages"
-
-join -1 1 -2 5 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/delta-packages" "${tmp_dir}/known-packages" >> \
- "${tmp_dir}/append-packages"
+ expand_version 2 | \
+ sort -k3,4 -k2Vr,2 -k1,1 | \
+ shrink_version 2 | \
+ uniq -f2 | \
+ sed -n '
+ s/^theirs \(\S\+ \)\{2\}//
+ T
+ p
+ ' | \
+ sort -u > \
+ "${tmp_dir}/must-haves"
+fi
+# shellcheck disable=SC2016
{
- awk '{print $5}' "${tmp_dir}/append-packages"
- cat "${tmp_dir}/delta-packages"
+ printf 'CREATE TEMPORARY TABLE `ignore_packages` (`pkgname` VARCHAR(64));\n'
+ if [ -s "${tmp_dir}/ignore-packages" ]; then
+ grep -vxF '' "${tmp_dir}/ignore-packages" | \
+ base64_encode_each | \
+ sed '
+ s/^/(from_base64("/
+ s/$/")),/
+ $s/,$/;/
+ 1 s/^/INSERT INTO `ignore_packages` (`pkgname`) VALUES \n/
+ '
+ fi
+ # packages on the build-list or deletion-list should be ignored
+ printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)'
+ printf ' SELECT DISTINCT `ignore_bin`.`pkgname`'
+ printf ' FROM `binary_packages` AS `ignore_bin`'
+ mysql_join_binary_packages_repositories 'ignore_bin'
+ printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list")'
+ # packages with no not-to-be-deleted and at least on to-be-deleted version should be ignored
+ printf ' OR ('
+ printf '`ignore_bin`.`is_to_be_deleted`'
+ printf ' AND NOT EXISTS ('
+ printf 'SELECT * FROM `binary_packages` AS `other_bin`'
+ printf ' WHERE NOT `other_bin`.`is_to_be_deleted`'
+ printf ' AND `other_bin`.`pkgname`=`ignore_bin`.`pkgname`'
+ printf '));\n'
} | \
- sort | \
- uniq -u | \
- sponge "${tmp_dir}/delta-packages"
+ sponge "${tmp_dir}/ignore-packages"
-if [ -s "${tmp_dir}/delta-packages" ]; then
- echo 'There are still packages newer for x86_64 which I cannot identify :-/'
- cat "${tmp_dir}/delta-packages"
- exit 2
-fi
-
-rm "${tmp_dir}/delta-packages"
-
-# convert append-packages to build-list.new (add git hashes)
+# shellcheck disable=SC2016
+{
+ printf 'CREATE TEMPORARY TABLE `must_haves` (`pkgname` VARCHAR(64));\n'
+ if [ -s "${tmp_dir}/must-haves" ]; then
+ grep -vxF '' "${tmp_dir}/must-haves" | \
+ base64_encode_each | \
+ sed '
+ s/^/(from_base64("/
+ s/$/")),/
+ $s/,$/;/
+ 1 s/^/INSERT INTO `must_haves` (`pkgname`) VALUES \n/
+ '
+ fi
+} | \
+ sponge "${tmp_dir}/must-haves"
-cut -d' ' -f1,4 < \
- "${tmp_dir}/append-packages" | \
- sort -u | \
- while read -r pkg repo; do
- git_repo=$(find_git_repository_to_package_repository "${repo}")
- printf '%s %s %s %s\n' \
- "${pkg}" \
- "$(cat "${work_dir}/${git_repo}.revision")" \
- "$(cat "${work_dir}/archlinux32.revision")" \
- "${repo}"
+# fetch unknown must-haves from upstream
+# shellcheck disable=SC2016
+printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` VARCHAR(64));\n' > \
+ "${tmp_dir}/pkgbases"
+# shellcheck disable=SC2016
+{
+ cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages"
+ printf 'SELECT `must_haves`.`pkgname` FROM `must_haves`'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `binary_packages`'
+ printf ' WHERE `binary_packages`.`pkgname`=`must_haves`.`pkgname`'
+ printf ') AND NOT EXISTS ('
+ printf 'SELECT * FROM `ignore_packages`'
+ printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`'
+ printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n'
+} | \
+ mysql_run_query | \
+ while read -r pkgname; do
+ content=$(
+ curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
+ tr ',' '\n'
+ )
+ repo=$(
+ printf '%s\n' "${content}" | \
+ sed -n '
+ s/^\s*"repo"\s*:\s*"//
+ T
+ s/".*$//
+ T
+ p
+ '
+ )
+ pkgbase=$(
+ printf '%s\n' "${content}" | \
+ sed -n '
+ s/^\s*"pkgbase"\s*:\s*"//
+ T
+ s/".*$//
+ T
+ p
+ '
+ )
+ if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
+ >&2 printf 'Could not find "%s" which is newer on x86_64!?\n' "${pkgname}"
+ exit 2
+ fi
+ printf '(from_base64("%s"),from_base64("%s")),\n' \
+ "$(printf '%s' "${pkgbase}" | base64 -w0)" \
+ "$(printf '%s' "${repo}" | base64 -w0)"
done | \
- sort -u > \
- "${tmp_dir}/build-list.append"
-
-# Create a lock file for build list.
-
-if ${update}; then
- # always block if locked
- exec 9> "${build_list_lock_file}"
- flock 9
-
- exec 8> "${sanity_check_lock_file}"
- flock -s 8
-fi
+ sort -u | \
+ sed '
+ 1 s/^/INSERT IGNORE INTO `pkgbases` (`pkgbase`,`repository`) VALUES \n/
+ $s/,$/;/
+ ' >> \
+ "${tmp_dir}/pkgbases"
+# shellcheck disable=SC2016
{
- awk '{print $2 " " $3 " " $4 " " $1}' "${tmp_dir}/build-list.append"
-
- # ignore packages on the build-list
- awk '{print $2 " " $3 " " $4 " " $1}' < \
- "${work_dir}/build-list" | \
- sed 'p'
-
- # ignore packages on the deletion-list
- awk '{print "X X X " $1}' "${work_dir}/deletion-list" | \
- sed 'p'
-
- # ignore packages on the blacklist
- git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
- tar -Ox blacklist | \
- sed '
- s/\s*#.*$//
- /^\s*$/d
- ' | \
- awk '{print "X X X " $1}' | \
- sed 'p'
-
- # ignore explicitely ignored packages
- awk '{print "X X X " $1}' "${tmp_dir}/ignore-packages" | \
- sed 'p'
+ cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages" "${tmp_dir}/pkgbases"
+ if ${auto}; then
+ printf 'CREATE TEMPORARY TABLE `bin_ids` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'INSERT IGNORE INTO `bin_ids`'
+ printf ' SELECT `binary_packages`.`id` FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ printf ' AND `repositories`.`name`!="build-support"'
+ mysql_join_binary_packages_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ # nothing "less stable" has the same name
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `binary_packages` AS `subst_bp`'
+ mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+ printf ' AND `subst_r`.`name`!="build-support"'
+ printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`subst_r`.`stability`'
+ printf ' WHERE `repository_stability_relations`.`more_stable`=`repositories`.`stability`'
+ printf ' AND `subst_bp`.`id`!=`binary_packages`.`id`'
+ printf ' AND `subst_bp`.`pkgname`=`binary_packages`.`pkgname`'
+ printf ') AND NOT EXISTS ('
+ printf 'SELECT * FROM `install_target_providers`'
+ mysql_join_install_target_providers_binary_packages '' 'subst_bp'
+ mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+ printf ' AND `subst_r`.`name` NOT IN ("build-support","deletion-list","to-be-decided")'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ')'
+ printf ';\n'
+ fi
+ printf 'SELECT '
+ printf '`pkgbases`.`pkgbase`,'
+ printf '`git_repositories`.`head`,'
+ printf '('
+ printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`'
+ printf ' WHERE `al32`.`name`="archlinux32"'
+ printf '),'
+ printf '`pkgbases`.`repository`'
+ printf ' FROM `pkgbases`'
+ printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`'
+ mysql_join_upstream_repositories_git_repositories
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `ignore_packages`'
+ printf ' JOIN `binary_packages` AS `i_bp` ON `ignore_packages`.`pkgname`=`i_bp`.`pkgname`'
+ mysql_join_binary_packages_build_assignments 'i_bp' 'i_ba'
+ mysql_join_build_assignments_package_sources 'i_ba' 'i_ps'
+ printf ' WHERE `i_ps`.`pkgbase`=`pkgbases`.`pkgbase`'
+ printf ');\n'
+ printf ';\n'
+ printf 'SELECT '
+ printf '`package_sources`.`pkgbase`,'
+ printf '`git_repositories`.`head`,'
+ printf '('
+ printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`'
+ printf ' WHERE `al32`.`name`="archlinux32"'
+ printf '),'
+ printf '`upstream_repositories`.`name`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_git_repositories
+ printf ' WHERE ('
+ if [ -s "${tmp_dir}/package-regexes" ]; then
+ grep -vxF '' "${tmp_dir}/package-regexes" | \
+ base64_encode_each | \
+ sed '
+ s/^/`binary_packages`.`pkgname` REGEXP from_base64("/
+ s/$/") OR /
+ ' | \
+ tr -d '\n'
+ fi
+ if ${auto}; then
+ printf 'EXISTS ('
+ printf 'SELECT * FROM `bin_ids`'
+ printf ' WHERE `bin_ids`.`id`=`binary_packages`.`id`'
+ printf ')'
+ printf ' OR '
+ fi
+ printf 'EXISTS ('
+ printf 'SELECT * FROM `must_haves`'
+ printf ' WHERE `must_haves`.`pkgname`=`binary_packages`.`pkgname`'
+ printf ')) AND NOT EXISTS ('
+ printf 'SELECT * FROM `ignore_packages`'
+ printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`'
+ printf ');\n'
} | \
- sort -k4,4 | \
- uniq -uf3 > \
- "${tmp_dir}/build-list.new"
-
-if ${update}; then
- awk '{print $4 " " $1 " " $2 " " $3}' \
- "${tmp_dir}/build-list.new" | \
- tee -a "${work_dir}/build-list"
-
- while read -r git_revision mod_git_revision repository pkgbase; do
- mysql_generate_package_metadata "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
- done < \
- "${tmp_dir}/build-list.new"
-
- # Remove the lock file
-
- rm -f "${build_list_lock_file}"
-else
- awk '{print $4 " " $1 " " $2 " " $3}' \
- "${tmp_dir}/build-list.new"
-fi
+ mysql_run_query | \
+ sort -u | \
+ tr '\t' ' ' | \
+ if ${update}; then
+ while read -r pkgbase git_rev mod_git_rev repo; do
+ printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
+ printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \
+ sed 's/ $/\n/'
+ mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
+ printf '\n' >&2
+ done
+
+ rm -f "${build_list_lock_file}"
+ else
+ cat
+ fi
diff --git a/bin/show-dependencies b/bin/show-dependencies
index efdf745..492f11b 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -3,85 +3,19 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
+# TODO: can this be faster?
+
# Create a lock file.
if [ $# -eq 0 ]; then
broken=$(
- find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
- sed 's|\(\.[^.]\+\)\{4\}$||' | \
- sort -u
- )
- new_sum='x'
- sum=''
- tmp_dir=$(mktemp -d 'tmp.show-dependencies.0.XXXXXXXXXX' --tmpdir)
- trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
- find "${work_dir}/package-infos" -maxdepth 1 -name '*.builds' -exec \
- grep -HF '' {} \; | \
- sed '
- s|^.*/||
- s|\(\.[^.]\+\)\{4\}:| |
- ' | \
- sort -k2,2 > \
- "${tmp_dir}/builds"
- while ! [ "${new_sum}" = "${sum}" ]; do
- sum="${new_sum}"
- # shellcheck disable=SC2086
- printf '%s\n' ${broken} | \
- sort > \
- "${tmp_dir}/broken"
- broken=$(
- {
- find "${work_dir}/package-infos" -maxdepth 1 -name '*.build-depends' | \
- sed 's|^.*/\(.*\)\(\.[^.]\+\)\{4\}$|\1 \0|' | \
- sort -k1,1 | \
- join -1 1 -2 1 -o 1.2 - "${tmp_dir}/broken" | \
- xargs -r cat
- } | \
- sort -u | \
- join -1 1 -2 2 -o 2.1 - "${tmp_dir}/builds"
- )
- broken=$(
- {
- {
- # shellcheck disable=SC2086
- printf '%s\n' ${broken}
- cat "${tmp_dir}/broken"
- } | \
- sort -u
- cut -d' ' -f1 < \
- "${work_dir}/build-list" | \
- sort -u
- } | \
- sort | \
- uniq -d
- )
- new_sum=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${broken} | \
- sha512sum
- )
- done
-
- rm -rf --one-file-system "${tmp_dir}"
- trap - EXIT
- broken=$(
- {
- # shellcheck disable=SC2086
- printf '%s\n' ${broken} 'ALL'
- {
- find "${webserver_directory}/graphs" -maxdepth 1 -name '*.png' -printf '%f\n' | \
- sed 's|\.png$||'
- {
- cut -d' ' -f1 < \
- "${work_dir}/build-list"
- cat "${work_dir}/deletion-list"
- } | \
- sort -u
- } | \
- sort | \
- uniq -d
- } | \
+ printf 'CALL show_broken_packages_and_dependencies;\n' | \
+ mysql_run_query | \
+ sed '
+ s/\s.*$//
+ ' | \
sort -u
+ printf 'ALL\n'
)
# shellcheck disable=SC2086
"$0" ${broken}
@@ -119,360 +53,248 @@ fi
tmp_dir=$(mktemp -d 'tmp.show-dependencies.1.XXXXXXXXXX' --tmpdir)
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-sort -u "${work_dir}/build-order" | \
- grep -v '^\(\S\+\) \1$' | \
- grep '^\S\+ \S\+$' > \
- "${tmp_dir}/original-build-order"
-
-sort -k1,1 "${work_dir}/build-list" > \
- "${tmp_dir}/build-list.sorted-by-package"
-
-{
- sort -k1,1 "${tmp_dir}/original-build-order" | \
- join -1 1 -2 1 -o 1.1,1.2 - "${tmp_dir}/build-list.sorted-by-package"
- sort -k2,2 "${tmp_dir}/original-build-order" | \
- join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/build-list.sorted-by-package"
-} | \
- sort -u | \
- sponge "${tmp_dir}/original-build-order"
-
-find "${work_dir}/package-infos" -maxdepth 1 -printf '%f\n' | \
- sed 's|\(\.[^.]\+\)\{4\}$||' | \
- sort -u > \
- "${tmp_dir}/known-packages"
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.groups' -printf '%f %p\n' | \
- while read -r name pf; do
- xargs -r printf "${name%.*.*.*.*}"' %s\n' < \
- "${pf}"
- done > \
- "${tmp_dir}/known-groups"
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f %p\n' | \
- while read -r name pf; do
- grep -vxF "${name%.*.*.*.*}" "${pf}" | \
- xargs -r printf "${name%.*.*.*.*}"' %s\n'
- done > \
- "${tmp_dir}/known-split-packages"
-
-find "${work_dir}/package-infos" -maxdepth 1 -name '*.run-depends' -printf '%f %p\n' | \
- while read -r name pf; do
- xargs -r printf '%s '"${name%.*.*.*.*}"'\n' < \
- "${pf}"
- done > \
- "${tmp_dir}/known-run-depends"
-
for target_package in "$@"; do
output="${webserver_directory}/graphs/${target_package}.png"
- if [ "${target_package}" = 'ALL' ]; then
-
- # groups and split packages built by jobs on the build list
- cat "${tmp_dir}/known-groups" "${tmp_dir}/known-split-packages" | \
- sort -k1,1 | \
- join -1 1 -2 1 -o 1.2 - "${tmp_dir}/build-list.sorted-by-package" | \
- sort -u > \
- "${tmp_dir}/relevant-stuff"
-
- {
- # groups and split packages built by jobs on the build list - which are also dependencies of packages on the build-list
- sort -k1,1 "${tmp_dir}/original-build-order" | \
- join -1 1 -2 1 -o 2.1 - "${tmp_dir}/relevant-stuff"
- # build list jobs themself
- cut -d' ' -f1 < "${work_dir}/build-list"
- } | \
- sort -u | \
- sponge "${tmp_dir}/relevant-stuff"
-
- {
- sort -k1,1 "${tmp_dir}/original-build-order" | \
- join -1 1 -2 1 -o 1.1,1.2 - "${tmp_dir}/relevant-stuff" | \
- sort -k2,2 | \
- join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/relevant-stuff"
- sed 's/.*/\0 \0/' "${tmp_dir}/relevant-stuff"
- } > \
- "${tmp_dir}/build-order"
-
- rm "${tmp_dir}/relevant-stuff"
-
- find "${work_dir}/package-states" -name '*.done' -printf '%f\n' | \
- sed 's|\(\.[^.]\+\)\{4\}$||' | \
- sort -u | \
- tee "${tmp_dir}/done-packages" | \
- sed 's|^|0 staging-package |' > \
- "${tmp_dir}/knots"
-
- {
- cat \
- "${tmp_dir}/known-groups" \
- "${tmp_dir}/known-run-depends" \
- "${tmp_dir}/known-split-packages"
- awk '{print $1 " " $1}' < \
- "${tmp_dir}/known-packages"
- } | \
- sort -k1,1 > \
- "${tmp_dir}/known-connections.sorted.1"
-
- sort -k2,2 \
- "${tmp_dir}/known-connections.sorted.1" > \
- "${tmp_dir}/known-connections.sorted.2"
-
- {
- # staging -> something -> build-list
- join -1 1 -2 1 -o 1.1,1.2 \
- "${tmp_dir}/known-connections.sorted.1" \
- "${tmp_dir}/done-packages" | \
- sort -k2,2 | \
- join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \
- - \
- "${tmp_dir}/known-connections.sorted.1" | \
- sort -k4,4 | \
- join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \
- - \
- "${tmp_dir}/build-list.sorted-by-package" | \
- sed '
- s|^\(\S\+ \S\+\) |\1\n|
- '
- # staging -> something -> staging
- join -1 1 -2 1 -o 1.1,1.2 \
- "${tmp_dir}/known-connections.sorted.1" \
- "${tmp_dir}/done-packages" | \
- sort -k2,2 | \
- join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \
- - \
- "${tmp_dir}/known-connections.sorted.1" | \
- sort -k4,4 | \
- join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \
- - \
- "${tmp_dir}/done-packages" | \
- sed '
- s|^\(\S\+ \S\+\) |\1\n|
- '
- # build-list -> something -> staging
- join -1 1 -2 1 -o 1.1,1.2 \
- "${tmp_dir}/known-connections.sorted.1" \
- "${tmp_dir}/build-list.sorted-by-package" | \
- sort -k2,2 | \
- join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \
- - \
- "${tmp_dir}/known-connections.sorted.1" | \
- sort -k4,4 | \
- join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \
- - \
- "${tmp_dir}/done-packages" | \
- sed '
- s|^\(\S\+ \S\+\) |\1\n|
- '
- cat "${tmp_dir}/build-order"
- } | \
- sort -u | \
- sponge "${tmp_dir}/build-order"
-
- rm \
- "${tmp_dir}/done-packages" \
- "${tmp_dir}/known-connections.sorted.1" \
- "${tmp_dir}/known-connections.sorted.2"
-
- else
-
- grep " $(str_to_regex "${target_package}")\$" "${tmp_dir}/original-build-order" | \
- sort -u > \
- "${tmp_dir}/build-order"
- printf '0 target-package %s\n' "${target_package}" > \
- "${tmp_dir}/knots"
-
- last_sum=''
- current_sum=$(sha512sum "${tmp_dir}/build-order")
- while ! [ "${last_sum}" = "${current_sum}" ]; do
-
- last_sum="${current_sum}"
-
- awk '{print $1}' "${tmp_dir}/build-order" | \
- sort -u > \
- "${tmp_dir}/new"
- {
- cat \
- "${tmp_dir}/original-build-order" \
- "${tmp_dir}/known-split-packages"
- sed '/ base\(-devel\)\?$/d' "${tmp_dir}/known-groups"
- } | \
- sort -k2,2 | \
- join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/new" | \
- sponge -a "${tmp_dir}/build-order"
-
- rm "${tmp_dir}/new"
-
- sort -u "${tmp_dir}/build-order" | \
- sponge "${tmp_dir}/build-order"
-
- current_sum=$(sha512sum "${tmp_dir}/build-order")
-
- done
-
- fi
+ # shellcheck disable=SC2016
+ {
+ printf 'CREATE TEMPORARY TABLE `relevant_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `relevant_binary_packages_copy` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `relevant_install_targets` (`id` BIGINT, UNIQUE KEY (`id`));\n'
-# grep -v '^\(\S\+\) \1$' "${tmp_dir}/build-order" | \
-# sponge "${tmp_dir}/build-order"
+ if [ "${target_package}" = 'ALL' ]; then
- # shellcheck disable=SC2129
- tr ' ' '\n' < \
- "${tmp_dir}/build-order" | \
- sort -u | \
- join -j 1 - "${tmp_dir}/build-list.sorted-by-package" | \
- while read -r pkg rev mod_rev repo; do
- if [ -f "${work_dir}/package-states/${pkg}.${rev}.${mod_rev}.${repo}.broken" ]; then
- printf '1 broken-build-list-package %s\n' "${pkg}"
- elif [ -f "${work_dir}/package-states/${pkg}.${rev}.${mod_rev}.${repo}.blocked" ]; then
- printf '2 blocked-build-list-package %s\n' "${pkg}"
- else
- printf '3 build-list-package %s\n' "${pkg}"
- fi
- done >> \
- "${tmp_dir}/knots"
+ printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)'
+ printf ' SELECT DISTINCT `binary_packages`.`id`'
+ printf ' FROM `repositories`'
+ mysql_join_repositories_repository_stabilities
+ printf ' AND `repository_stabilities`.`name` IN ("unbuilt","staging")'
+ mysql_join_repositories_binary_packages
+ printf ';\n'
- {
- sort -u "${work_dir}/deletion-list"
- tr ' ' '\n' < \
- "${tmp_dir}/build-order" | \
- sort -u
- } | \
- sort | \
- uniq -d | \
- xargs -r printf '4 deletion-list-package %s\n' >> \
- "${tmp_dir}/knots"
+ else
- {
- awk '{print $2}' "${tmp_dir}/known-split-packages" | \
- sort -u
- tr ' ' '\n' < \
- "${tmp_dir}/build-order" | \
- sort -u
- } | \
- sort | \
- uniq -d | \
- xargs -r printf '5 split-package %s\n' >> \
- "${tmp_dir}/knots"
+ printf 'CALL calculate_dependencies_of_package_upto_first_built_one(from_base64("%s"));\n' \
+ "$(
+ printf '%s' "${target_package}" | \
+ base64 -w0
+ )"
- {
- cat "${tmp_dir}/known-packages"
- tr ' ' '\n' < \
- "${tmp_dir}/build-order" | \
- sort -u
- } | \
- sort | \
- uniq -d | \
- xargs -r printf '6 package %s\n' >> \
- "${tmp_dir}/knots"
+ fi
- {
- awk '{print $2}' "${tmp_dir}/known-groups" | \
- sort -u
- tr ' ' '\n' < \
- "${tmp_dir}/build-order" | \
- sort -u
+ printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)'
+ printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`'
+ printf ';\n'
+ printf 'INSERT IGNORE INTO `relevant_install_targets` (`id`)'
+ printf ' SELECT DISTINCT `install_target_providers`.`install_target`'
+ printf ' FROM `relevant_binary_packages`'
+ mysql_join_binary_packages_install_target_providers 'relevant_binary_packages'
+ mysql_join_install_target_providers_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_building`'
+ printf ' JOIN `relevant_binary_packages_copy` ON `relevant_binary_packages_copy`.`id`=`dependencies`.`dependent`'
+ printf ';\n'
+
+ # we return lines with either:
+ # "knot" $type $identifier $label
+ # or
+ # "edge" $type $from_knot $to_knot
+ printf 'SELECT DISTINCT'
+ printf ' "knot",'
+ printf 'IF(`build_assignments`.`is_%s`,"%s-build-list-pkgbase",' \
+ 'blocked' 'broken' \
+ 'broken' 'broken'
+ printf '"build-list-pkgbase"'
+ printf ')),'
+ printf 'CONCAT("pkgbase:",`package_sources`.`id`),'
+ printf '`package_sources`.`pkgbase`'
+ printf ' FROM `relevant_binary_packages`'
+ printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' AND `repository_stabilities`.`name`="unbuilt"'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ printf ';\n'
+
+ printf 'SELECT DISTINCT'
+ printf ' "knot",'
+ printf 'CONCAT("pkgname-",`repository_stabilities`.`name`),'
+ printf 'CONCAT("pkgname:",`binary_packages`.`id`),'
+ printf '`binary_packages`.`pkgname`'
+ printf ' FROM `relevant_binary_packages`'
+ printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ';\n'
+
+ printf 'SELECT DISTINCT'
+ printf ' "knot","install-target",'
+ printf 'CONCAT("install-target:",`install_targets`.`id`),'
+ printf '`install_targets`.`name`'
+ printf ' FROM `relevant_install_targets`'
+ printf ' JOIN `install_targets` ON `install_targets`.`id`=`relevant_install_targets`.`id`'
+ printf ';\n'
+
+ printf 'SELECT DISTINCT'
+ printf ' "edge","builds"'
+ printf ',CONCAT("%s:",`%s`.`id`)' \
+ 'pkgbase' 'package_sources' \
+ 'pkgname' 'binary_packages'
+ printf ' FROM `relevant_binary_packages`'
+ printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' AND `repository_stabilities`.`name`="unbuilt"'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ printf ';\n'
+
+ printf 'SELECT DISTINCT'
+ printf ' "edge","provides"'
+ printf ',CONCAT("%s:",`%s`.`id`)' \
+ 'pkgname' 'binary_packages' \
+ 'install-target' 'install_targets'
+ printf ' FROM `relevant_binary_packages`'
+ printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_install_target_providers
+ printf ' JOIN `relevant_install_targets` ON `relevant_install_targets`.`id`=`install_target_providers`.`install_target`'
+ mysql_join_install_target_providers_install_targets
+ printf ';\n'
+
+ printf 'SELECT DISTINCT'
+ printf ' "edge",CONCAT("depends:",`dependency_types`.`name`)'
+ printf ',CONCAT("%s:",`%s`.`id`)' \
+ 'install-target' 'install_targets' \
+ 'pkgname' 'binary_packages'
+ printf ' FROM `relevant_binary_packages`'
+ printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_dependencies
+ printf ' JOIN `relevant_install_targets` ON `relevant_install_targets`.`id`=`dependencies`.`depending_on`'
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_building`'
+ mysql_join_dependencies_install_targets
+ printf ';\n'
} | \
- sort | \
- uniq -d | \
- xargs -r printf '7 group %s\n' >> \
- "${tmp_dir}/knots"
-
- tr ' ' '\n' < \
- "${tmp_dir}/build-order" | \
- sort -u | \
- xargs -r printf '100 unknown %s\n' >> \
+ mysql_run_query | \
+ sed '
+ y/\t/ /
+ /^knot /{
+ s/^\S\+ //
+ w '"${tmp_dir}"'/knots
+ d
+ }
+ /^edge /{
+ s/^\S\+ //
+ w '"${tmp_dir}"'/edges
+ d
+ }
+ ' >&2
+
+ mkdir "${tmp_dir}/find-identical"
+
+ awk '{
+ print $1 >> "'"${tmp_dir}"'/find-identical/characteristics-"$2
+ }' < \
"${tmp_dir}/knots"
+ awk '{
+ print $1 " X " $3 >> "'"${tmp_dir}"'/find-identical/characteristics-"$2
+ print $1 " " $2 " X" >> "'"${tmp_dir}"'/find-identical/characteristics-"$3
+ }' < \
+ "${tmp_dir}/edges"
+
+ modifier=$(
+ find "${tmp_dir}/find-identical" -maxdepth 1 \
+ -type f \
+ -name 'characteristics-*' | \
+ while read -r file; do
+ printf '%s ' "${file##*/}"
+ sort -u "${file}" | \
+ sha512sum | \
+ awk '{print $1}'
+ done | \
+ sort -k2,2 | \
+ uniq -Df1 | \
+ sed 's,^characteristics-,,' | \
+ sed '
+ :a
+ $!N
+ s/^\(\S\+\) \(\S\+\)\n\(\S\+ \)\2$/\1+\3\2/
+ ta
+ P
+ D
+ ' | \
+ cut -d' ' -f1 | \
+ sed '
+ s/^/s,\\(/
+ s/+\([^+[:space:]]\+\)$/\\)\\(\\s\\|$\\),\1\\2,g;/
+ s/+/\\|/g
+ '
+ )
+ rm -rf --one-file-system "${tmp_dir}/find-identical"
- sort -k3,3 -k1n,1 "${tmp_dir}/knots" | \
- uniq -f2 | \
- awk '{print $3 " " $1 " " $2}' | \
- sort -k2,3 | \
+ sed "${modifier}" "${tmp_dir}/knots" | \
+ sort -k2,2 -k1,1 | \
+ sed '
+ :a
+ $!N
+ s/^\(\S\+ \S\+ \)\(\S\+\)\n\1\(\S\+\)$/\1\2<nl>\3/
+ ta
+ P
+ D
+ ' | \
sponge "${tmp_dir}/knots"
- mkdir "${tmp_dir}/neighbours"
- cat "${tmp_dir}/knots" "${tmp_dir}/build-order" | \
- awk '{
- print $1 " " $2 > "'"${tmp_dir}/neighbours/"'"$1;
- print $1 " " $2 > "'"${tmp_dir}/neighbours/"'"$2;
- }'
-
- find "${tmp_dir}/neighbours" -maxdepth 1 -type f | \
- while read -r file; do
- sed '
- s@\(^\| \)'"$(str_to_regex "${file##*/}")"'\( \|$\)@\1<knot>\2@
- s@\(^\| \)'"$(str_to_regex "${file##*/}")"'\( \|$\)@\1<knot>\2@
- ' "${file}" | \
- sort -u | \
- sponge "${file}"
- done
-
- find "${tmp_dir}/neighbours" -maxdepth 1 -type f -exec \
- sha512sum {} \; | \
- sed 's|^\(\S\+\) .*/\([^/]\+\)$|\2 \1|' | \
- sort -k2,2 | \
- uniq -f1 -D | \
- awk '{print $2 " " $1}' > \
- "${tmp_dir}/sums"
-
- rm -rf --one-file-system "${tmp_dir}/neighbours"
-
- sed '
- :a
- $!N
- s|^\(\S\+\) \([^\n]\+\)\n\1 |\1 \2<nl>|
- ta
- P
- D
- ' "${tmp_dir}/sums" | \
- join -1 1 -2 1 -o 2.2,1.2 - "${tmp_dir}/sums" | \
- while read -r original replacement; do
- sed -i '
- s@\( \|^\)'"$(str_to_regex "${original}")"'\( \|$\)@\1'"${replacement}"'\2@
- s@\( \|^\)'"$(str_to_regex "${original}")"'\( \|$\)@\1'"${replacement}"'\2@
- ' "${tmp_dir}/build-order" "${tmp_dir}/knots"
- done
-
- grep -v '^\(\S\+\) \1$' "${tmp_dir}/build-order" | \
+ sed "${modifier}" "${tmp_dir}/edges" | \
sort -u | \
- sponge "${tmp_dir}/build-order"
- sort -u "${tmp_dir}/knots" | \
- sponge "${tmp_dir}/knots"
+ sponge "${tmp_dir}/edges"
{
printf '%s\n' \
'digraph dependencies {' \
' fontname=dejavu;'
- cut -d' ' -f1,3 < \
- "${tmp_dir}/knots" | \
- while read -r who what; do
- if [ "${what}" = 'broken-build-list-package' ]; then
+ # knots: $type $identifier $label
+ # edges: $type $from_knot $to_knot
+
+ while read -r type id label; do
+ case "${type}" in
+ 'broken-build-list-pkgbase')
color='#ff0000'
- elif [ "${what}" = 'blocked-build-list-package' ]; then
+ ;;
+ 'blocked-build-list-pkgbase')
color='#d00000'
- elif [ "${what}" = 'build-list-package' ]; then
+ ;;
+ 'build-list-pkgbase')
color='#800000'
- elif [ "${what}" = 'deletion-list-package' ]; then
- color='#808080'
- elif [ "${what}" = 'group' ]; then
- color='#0000ff'
- elif [ "${what}" = 'package' ]; then
- color='#000000'
- elif [ "${what}" = 'split-package' ]; then
- color='#8080ff'
- elif [ "${what}" = 'staging-package' ]; then
+ ;;
+ 'pkgname-unbuilt')
+ color='#800000'
+ ;;
+ 'pkgname-staging')
color='#008000'
- elif [ "${what}" = 'target-package' ]; then
- color='#00ff00'
- else
+ ;;
+ 'pkgname-stable'|'pkgname-testing'|'pkgname-standalone')
+ color='#000000'
+ ;;
+ 'install-target')
+ color='#000080'
+ ;;
+ *)
color='#ff80ff'
- fi
- printf ' "%s" [fontcolor="%s"];\n' "${who}" "${color}"
- done
+ label="${label} (${type})"
+ ;;
+ esac
+ printf ' "%s" [label="%s", fontcolor="%s"];\n' "${id}" "${label}" "${color}"
+ done < \
+ "${tmp_dir}/knots"
- sed 's|\\|\\\\|g' "${tmp_dir}/build-order" | \
- xargs -rn2 printf ' "%s" -> "%s";\n' | \
- sort -u
+ while read -r type from_knot to_knot; do
+ printf '"%s" -> "%s";\n' \
+ "${from_knot}" "${to_knot}"
+ done < \
+ "${tmp_dir}/edges"
printf '%s\n' \
'}'
@@ -486,16 +308,12 @@ for target_package in "$@"; do
' "${tmp_dir}/input"
line_count=$(wc -l < "${tmp_dir}/input")
- if [ "${line_count}" -gt 500 ] && [ "${target_package}" != 'ALL' ]; then
- sed -i '/"base\(-devel\)\?"/d' "${tmp_dir}/input"
- line_count=$(wc -l < "${tmp_dir}/input")
- if [ "${line_count}" -gt 700 ]; then
- rm -f "${output}"
- >&2 printf 'Skipping graph for "%s" - would be too big (%d).\n' \
- "${target_package}" \
- "${line_count}"
- continue
- fi
+ if [ "${target_package}" != 'ALL' ] && [ "${line_count}" -gt 1000 ]; then
+ rm -f "${output}"
+ >&2 printf 'Skipping graph for "%s" - would be too big (%d).\n' \
+ "${target_package}" \
+ "${line_count}"
+ continue
fi
printf 'small enough (%s): %d\n' \
"${target_package}" \
diff --git a/bin/slave-build-connect b/bin/slave-build-connect
index 2ca72a3..f8ee9ad 100755
--- a/bin/slave-build-connect
+++ b/bin/slave-build-connect
@@ -3,11 +3,12 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
-if [ "${SSH_ORIGINAL_COMMAND%% *}" = "get-assignment" ] || \
- [ "${SSH_ORIGINAL_COMMAND%% *}" = "return-assignment" ]; then
+if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \
+ [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xreturn-assignment' ] || \
+ [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xping-from-slave' ]; then
# small check to prevent some shell-injections
- if echo "${SSH_ORIGINAL_COMMAND}" | \
+ if printf '%s\n' "${SSH_ORIGINAL_COMMAND}" | \
grep -q '[^-a-zA-Z0-9.+_ ]'; then
>&2 printf 'Invalid command: "%s".\n' "${SSH_ORIGINAL_COMMAND}"
@@ -28,7 +29,7 @@ if [ "${SSH_ORIGINAL_COMMAND%% *}" = "get-assignment" ] || \
base64 -w0
)"
} | \
- ${mysql_command}
+ mysql_run_query
slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}"
diff --git a/bin/strict-bashism-check b/bin/strict-bashism-check
index 3bb3884..cbf7e5b 100755
--- a/bin/strict-bashism-check
+++ b/bin/strict-bashism-check
@@ -49,37 +49,54 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
git archive "${tree}" | \
tar -C "${tmp_dir}" -x
+if ! cd "${tmp_dir}"; then
+ echo 'Cannot cd.'
+ exit 1
+fi
+
errors=$(
- cd "${tmp_dir}" || \
- echo 'Cannot cd.'
- find "bin" "conf" -type f -not -executable -not -name '.gitignore'
+ find bin conf lib -type f -not -executable -not -name '.gitignore'
)
if [ -n "${errors}" ]; then
- >&2 echo 'Non-executable files found in bin/ or conf/:'
+ >&2 echo 'Non-executable files found in bin/, conf/ or lib/:'
>&2 echo "${errors}"
exit 1
fi
errors=$(
-# shellcheck disable=SC2016
- find bin conf -type f -executable -exec grep -H '="\$(' {} \;
+ grep -r '\(\s\|^\)mysql buildmaster\($\|\s\)' bin lib conf
)
if [ -n "${errors}" ]; then
- >&2 echo 'Unnecessary quotes found:'
+ >&2 echo 'Style error: call "mysql_run_query" instead of "mysql buildmaster":'
>&2 echo "${errors}"
exit 1
fi
errors=$(
- cd "${tmp_dir}" || \
- echo 'Cannot cd.'
- shellcheck -x bin/* conf/* 2>&1
+# shellcheck disable=SC2016
+ find bin conf lib -type f -executable -exec grep -H '="\$(' {} \;
)
if [ -n "${errors}" ]; then
- >&2 echo 'shellcheck complains about the following:'
+ >&2 echo 'Unnecessary quotes found:'
>&2 echo "${errors}"
exit 1
fi
+
+if which shellcheck >/dev/null 2>&1; then
+ errors=$(
+ find bin conf lib \
+ -type f \
+ -not -name 'opcode_list' \
+ -not -name '.*' \
+ -exec shellcheck -x '{}' \; 2>&1
+ )
+
+ if [ -n "${errors}" ]; then
+ >&2 echo 'shellcheck complains about the following:'
+ >&2 echo "${errors}"
+ exit 1
+ fi
+fi
diff --git a/bin/why-dont-you b/bin/why-dont-you
index 2e3ea86..e873c71 100755
--- a/bin/why-dont-you
+++ b/bin/why-dont-you
@@ -7,6 +7,9 @@
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
+# TODO: reintrocude "keep", "stubbornly_keep", "stabilize" and "unstage"
+# using information from the database.
+
action="$1"
shift
@@ -17,342 +20,164 @@ case "${action}" in
'build')
- for pkg in "$@"; do
- {
- grep "^$(str_to_regex "${pkg}") " "${work_dir}/build-list" || \
- >&2 printf '"%s" is not on the build list.\n' "${pkg}"
- } | \
- while read -r package git_revision mod_git_revision repository; do
-
- if [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
- printf '"%s" is locked by ' "$1"
- sort -u < \
- "${work_dir}/package-states/$1.$2.$3.$4.locked" \
- sed '
- :a
- $!{
- N
- s/\n/, /
- ba
- }
- s/$/./
- '
- continue
- fi
- if [ -f "${work_dir}/package-states/$1.$2.$3.$4.blocked" ]; then
- printf '"%s" is blocked: "' "${pkg}"
- tr '[:space:]' ' ' < \
- "${work_dir}/package-states/$1.$2.$3.$4.blocked" | \
- sed '
- s| \+| |
- s|^ ||
- s| $||
- '
- printf '"\n'
- continue
- fi
-
- unmet_dependencies=$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")
- if [ -n "${unmet_dependencies}" ]; then
- printf '"%s" has unmet dependencies:\n' "${package}"
- echo "${unmet_dependencies}" | \
- while read -r dep; do
- grep -lxF "${dep}" "${work_dir}/package-infos/"*".builds" | \
- sed '
- s|^.*/||
- s|\(\.[^.]\+\)\{4\}||
- '
- done | \
- sort -u
- printf '\n'
-
- continue
- fi
-
- if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then
- printf '"%s" is broken (%sx built), but would be built\n' \
- "${pkg}" \
- "$(wc -l < "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken")"
- else
- printf '"%s" would be built\n' "${pkg}"
- fi
- done
-
- done
-
- ;;
-
- 'stabilize'|'unstage')
-
- if [ "${action}" = 'stabilize' ]; then
- suffix='tested'
- else
- suffix='done'
- fi
-
+ # shellcheck disable=SC2016
{
- awk '{print $1 "." $2 "." $3 "." $4}' < \
- "${work_dir}/build-list"
- if [ "${action}" = 'stabilize' ]; then
- find "${work_dir}/package-states" -maxdepth 1 \( \
- -name '*.done' -o \
- -name '*.testing' \
- \) -printf '%f\n' | \
- sed 's|\.[^.]\+$||'
- fi
+ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64));\n'
+ printf 'INSERT INTO `pkgbases` VALUES '
+ # shellcheck disable=SC2046
+ printf '(from_base64("%s")),' \
+ $(
+ printf '%s\n' "$@" | \
+ base64_encode_each
+ ) | \
+ sed 's/,$/;\n/'
+ # we select everything which is possibly of any interest:
+ # - id (to see if it actually is on the build-list
+ # - to_build.is_broken
+ # - failed_builds_count
+ # - to_build.is_blocked
+ # - deps.pkgbase (any dependency pending?)
+ # - build_slaves.name (is anyone building this?)
+ # - pkgbase
+ printf 'SELECT DISTINCT `to_build`.`ba_id`,'
+ printf 'If(`to_build`.`is_broken`,1,0),'
+ printf '('
+ printf 'SELECT count(*) FROM `failed_builds`'
+ printf 'WHERE `failed_builds`.`build_assignment`=`to_build`.`ba_id`'
+ printf ')'
+ printf ',replace(to_base64(`%s`.`%s`),"\\n","")' \
+ 'to_build' 'is_blocked' \
+ 'deps' 'pkgbase' \
+ 'build_slaves' 'name' \
+ 'pkgbases' 'pkgbase'
+ # at least one row for each given `pkgbase`
+ printf ' FROM `pkgbases`'
+ printf ' LEFT JOIN '
+ printf '('
+ # join the tables for the to-be-built packages:
+ # package_source, build_assignment, binary_package, repostory
+ printf 'SELECT DISTINCT `tb_ps`.`pkgbase`,`tb_bin`.`id` AS `bin_id`,`tb_ba`.`id` AS `ba_id`,`tb_ba`.`is_blocked`,`tb_ba`.`is_broken`'
+ printf ' FROM `package_sources` AS `tb_ps`'
+ mysql_join_package_sources_build_assignments 'tb_ps' 'tb_ba'
+ mysql_join_build_assignments_binary_packages 'tb_ba' 'tb_bin'
+ mysql_join_binary_packages_repositories 'tb_bin' 'tb_rep'
+ printf ' WHERE `tb_rep`.`name`="build-list"'
+ printf ') AS `to_build`'
+ printf ' ON `to_build`.`pkgbase`=`pkgbases`.`pkgbase`'
+ printf ' LEFT JOIN '
+ printf '('
+ # same join as above, but with different names - for the
+ # potential dependencies
+ printf 'SELECT DISTINCT `dep_ps`.`pkgbase`,`dependencies`.`dependent`'
+ printf ' FROM `package_sources` AS `dep_ps`'
+ mysql_join_package_sources_build_assignments 'dep_ps' 'dep_ba'
+ mysql_join_build_assignments_binary_packages 'dep_ba' 'dep_bin'
+ mysql_join_binary_packages_repositories 'dep_bin' 'dep_rep'
+ # now we have some (=3) additional joins,
+ # because we are interested in dependency relations to `to_build`
+ mysql_join_binary_packages_install_target_providers 'dep_bin'
+ mysql_join_install_target_providers_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' WHERE `dep_rep`.`name`="build-list"'
+ printf ' AND `dependency_types`.`relevant_for_building`'
+ printf ') AS `deps`'
+ printf ' ON `deps`.`dependent`=`to_build`.`bin_id`'
+ # now we join with build slaves to see if someone builds this
+ printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`'
+ printf ';\n'
} | \
- sort -u > \
- "${tmp_dir}/unmoveable-list"
-
- find "${work_dir}/package-states" -maxdepth 1 -name "*.${suffix}" -printf '%f\n' | \
- sed 's|\.[^.]\+$||' | \
- sort -u > \
- "${tmp_dir}/moveable-list"
-
- cat "${tmp_dir}/moveable-list" "${tmp_dir}/unmoveable-list" | \
+ mysql_run_query | \
+ tr '\t' ' ' | \
+ sort -k7,7 -k6,6 -k5,5 | \
sed '
- s|^|'"${work_dir}/package-infos/"'|
- s|$|.run-depends|
+ / NULL \S\+$/ b multi-dep
+ :multi-slave
+ $!N
+ s/^\(\(\S\+ \)\{5\}\)\(\S\+\)\( \S\+\)\n\(\S\+ \)\{5\}\(\S\+\)\4/\1\3,\6\4/
+ t multi-slave
+ P
+ D
+ :multi-dep
+ / NULL\( \S\+\)\{3\}$/! b
+ $!N
+ s/^\(\(\S\+ \)\{4\}\)\(\S\+\)\(\( \S\+\)\{2\}\)\n\(\S\+ \)\{4\}\(\S\+\)\4/\1\3,\7\4/
+ t multi-dep
+ P
+ D
' | \
- # base is boring, so we ignore it (might give result "... can be unstaged" although it _will_not_ be unstaged!
- xargs -r grep -vHxF 'base' | \
sed '
- s|^[^:]*/||
- s|\.run-depends:| |
+ s/NULL,//g
' | \
- sort -k2,2 > \
- "${tmp_dir}/all-run-depends"
-
- cat "${tmp_dir}/moveable-list" "${tmp_dir}/unmoveable-list" | \
- sed '
- s|^|'"${work_dir}/package-infos/"'|
- s|$|.builds|
- ' | \
- # base is boring, so we ignore it (might give result "... can be unstaged" although it _will_not_ be unstaged!
- xargs -r grep -vHxF 'base' | \
- sed '
- s|^[^:]*/||
- s|\.builds:| |
- ' | \
- sort -k2,2 > \
- "${tmp_dir}/all-builds"
-
- for pkg in "$@"; do
-
- if ! state_file=$(
- grep '^'"$(str_to_regex "${pkg}")"'\(\.[^.]\+\)\{3\}$' "${tmp_dir}/moveable-list"
- ) || \
- [ ! -f "${work_dir}/package-states/${state_file}.${suffix}" ]; then
- printf '"%s" is not %s yet!\n' "${pkg}" "${suffix}"
- continue
- fi
-
- echo "${state_file}" > "${tmp_dir}/dependent.new"
- touch "${tmp_dir}/dependent"
-
- while [ -s "${tmp_dir}/dependent.new" ]; do
- cat "${tmp_dir}/dependent.new" "${tmp_dir}/dependent" | \
- sort -u | \
- sponge "${tmp_dir}/dependent"
-
- {
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.builds|
- ' "${tmp_dir}/dependent.new" | \
- xargs -r cat | \
- sort -u | \
- join -1 1 -2 2 -o 2.1 - "${tmp_dir}/all-run-depends"
- sed '
- s|^|'"${work_dir}"'/package-infos/|
- s|$|.run-depends|
- ' "${tmp_dir}/dependent.new" | \
- xargs -r cat | \
- sort -u | \
- join -1 1 -2 2 -o 2.1 - "${tmp_dir}/all-builds"
- } | \
- sort -u | \
- sponge "${tmp_dir}/dependent.new"
-
- cat "${tmp_dir}/dependent.new" "${tmp_dir}/dependent" "${tmp_dir}/dependent" | \
- sort | \
- uniq -u | \
- sponge "${tmp_dir}/dependent.new"
- done
-
- dependent_unmoveable=$(
- join -1 1 -2 1 -o 2.1 "${tmp_dir}/unmoveable-list" "${tmp_dir}/dependent"
- )
-
- if [ -n "${dependent_unmoveable}" ]; then
- printf 'The following packages are dependent on "%s", but cannot be %sd:\n' "${pkg}" "${action}"
- echo "${dependent_unmoveable}" | \
- while read -r sf; do
- printf '%s' "${sf}"
- if [ -f "${work_dir}/package-states/${sf}.testing" ]; then
- printf ' (not tested yet for %s days)' "$(( ($(date '+%s') - $(stat -c '%Y' "${work_dir}/package-states/${sf}.testing")) / 3600 / 24 ))"
- elif [ -f "${work_dir}/package-states/${sf}.done" ]; then
- printf ' (not unstaged yet for %s days)' "$(( ($(date '+%s') - $(stat -c '%Y' "${work_dir}/package-states/${sf}.done")) / 3600 / 24 ))"
- elif tr ' ' '.' < \
- "${work_dir}/build-list" | \
- grep -qxF "${sf}"; then
- printf ' (not built yet)'
- fi
- printf '\n'
- done
- printf '\n'
- continue
- fi
-
- printf 'Package "%s" can be %sd.\n' "${pkg}" "${action}"
-
- done
-
- ;;
-
- 'keep'|'stubbornly_keep')
-
- find '/var/lib/pacman/sync' -name '*.db' -execdir bsdtar -tf '{}' \; | \
- sed -n '
- s|-[^-]\+-[^-]\+/$||
- T
- p
- ' | \
- sort -u > \
- "${tmp_dir}/upstream-packages"
-
- while read -r pkg; do
-
- if builds_file=$(
- find "${work_dir}/package-infos" -maxdepth 1 -printf '%f\n' | \
- grep -m1 '^'"$(str_to_regex "${pkg}")"'\(\.[^.]\+\)\{3\}\.builds$'
- ); then
-
- builds_file="${builds_file%.*}"
- prepo="${builds_file##*.}"
- builds_file="${builds_file%.*}"
- mod_rev="${builds_file##*.}"
- builds_file="${builds_file%.*}"
- rev="${builds_file##*.}"
-
- else
-
- found_PKGBUILD=false
- mod_rev=$(cat "${work_dir}/archlinux32.revision")
- for repo in ${repo_names}; do
- eval 'repo_path="${repo_paths__'"${repo}"'}"'
- rev=$(cat "${work_dir}/${repo}.revision")
- if [ "${repo}" = 'archlinux32' ]; then
- if git -C "${repo_path}" archive "${mod_rev}" | \
- grep -q '^[^/]\+/'"$(str_to_regex "${pkg}")"'/PKGBUILD$'; then
- prepo=$(
- git -C "${repo_path}" archive "${mod_rev}" | \
- grep '^[^/]\+/'"$(str_to_regex "${pkg}")"'/PKGBUILD$' | \
- cut -d/ -f1
- )
- found_PKGBUILD=true
- break
- fi
- else
- prepo=$(
- git -C "${repo_path}" archive "${rev}" -- "${pkg}/repos" 2>/dev/null | \
- tar -t 2> /dev/null | \
- grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
- grep -v -- '-i686/PKGBUILD$' | \
- grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
- sed '
- s|^[^/]\+/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1|
- ' | \
- head -n1
- )
- if [ -n "${prepo}" ]; then
- found_PKGBUILD=true
- break
- fi
- fi
- done
- if ! ${found_PKGBUILD}; then
+ while read -r id is_broken trials is_blocked dependency slave pkgbase; do
+ pkgbase=$(
+ printf '%s' "${pkgbase}" | \
+ base64 -d
+ )
+ if [ "${id}" = 'NULL' ]; then
+ >&2 printf '"%s" is not on the build list.\n' \
+ "${pkgbase}"
continue
fi
-
- generate_package_metadata "${pkg}" "${rev}" "${mod_rev}" "${prepo}"
-
- fi
-
- sed "s|^|${pkg} builds |" "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${prepo}.builds"
-
- done < \
- "${work_dir}/deletion-list" > \
- "${tmp_dir}/deleted.builds"
-
- sort -k3,3 "${tmp_dir}/deleted.builds" | \
- sponge "${tmp_dir}/deleted.builds"
-
- for pkg in "$@"; do
-
- if ! grep -qxF "${pkg}" "${work_dir}/deletion-list"; then
- printf 'Package "%s" is not on the deletion list.\n' "${pkg}"
- continue
- fi
-
- if ! grep -qxF "${pkg}" "${tmp_dir}/upstream-packages"; then
- printf 'Package "%s" is not available upstream.\n' "${pkg}"
- if [ "${action}" = 'keep' ]; then
+ if [ "${slave}" != 'NULL' ]; then
+ # beware: A slave named "5BË" will look exactly like this!
+ printf '"%s" is locked by %s.\n' \
+ "${pkgbase}" \
+ "$(
+ printf '%s\n' "${slave}" | \
+ tr ',' '\n' | \
+ while read -r line; do
+ printf '%s\n' "${line}" | \
+ base64 -d
+ printf ','
+ done | \
+ sed 's/,$//'
+ )"
continue
fi
- fi
-
- if git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
- tar -Ox 'blacklist' | \
- sed '
- s/\s*#.*$//
- /^\s*$/d
- ' | \
- grep -qxF "${pkg}"; then
- printf 'Package "%s" is explicitely blacklisted.\n' "${pkg}"
- continue
- fi
+ if [ "${is_blocked}" != 'NULL' ]; then
+ # beware: A block-reason "5BË" will look exactly like this!
+ printf '"%s" is blocked: "%s".\n' \
+ "${pkgbase}" \
+ "$(
+ printf '%s' "${is_blocked}" | \
+ base64 -d
+ )"
+ continue
+ fi
+ if [ "${dependency}" != 'NULL' ]; then
+ printf '"%s" has unmet dependencies:\n' \
+ "${pkgbase}"
+ printf '%s\n' "${dependency}" | \
+ tr ',' '\n' | \
+ while read -r line; do
+ printf ' '
+ printf '%s\n' "${line}" | \
+ base64 -d
+ printf '\n'
+ done
+ continue
+ fi
+ if [ "${is_broken}" = '1' ]; then
+ printf '"%s" is broken (%sx built), but would be built.\n' \
+ "${pkgbase}" \
+ "${trials}"
+ continue
+ fi
+ printf '"%s" would be built.\n' \
+ "${pkgbase}"
+ done
- if [ "lib32-${pkg#lib32-}" = "${pkg}" ]; then
- printf 'Package "%s" is a library from multilib.\n' "${pkg}"
- continue
- fi
+ ;;
- build_depends=$(
- find "${work_dir}/package-infos" -maxdepth 1 -name "${pkg}.*.build-depends" -exec cat {} \;
- )
- if [ -z "${build_depends}" ]; then
- printf 'Package "%s" was deleted in the git repositories.\n' "${pkg}"
- continue
- fi
+ 'stabilize'|'unstage')
- build_depends=$(
- echo "${build_depends}" | \
- sort -u
- )
+ printf 'Sry, "why-dont-you %s" is unavailable, until someone recodes it to look into the database.\n' "${action}"
- errors=$(
- {
- # shellcheck disable=SC2086
- printf '%s\n' ${build_depends}
- awk '{print $3}' "${tmp_dir}/deleted.builds" | \
- sort -u
- } | \
- sort | \
- uniq -d | \
- join -1 1 -2 3 -o 2.1,2.2,2.3 - "${tmp_dir}/deleted.builds"
- )
- if [ -n "${errors}" ]; then
- printf 'Package "%s" has dependencies on the deletion list:\n' "${pkg}"
- # shellcheck disable=SC2086,SC2183
- printf '%s %s %s\n' ${errors}
- printf '\n'
- continue
- fi
+ ;;
- printf 'It seems, package "%s" should not be deleted.\n' "${pkg}"
+ 'keep'|'stubbornly_keep')
- done
+ printf 'Sry, "why-dont-you %s" is unavailable, until someone recodes it to look into the database.\n' "${action}"
;;