summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2018-09-03 14:22:18 +0200
committerErich Eckner <git@eckner.net>2018-09-03 14:22:18 +0200
commitdb5448138b313407d8062c04ed6df458ff64c66f (patch)
tree290a3e6783b581b231ce919b573043e90ad39061
parent385150b196c418d3056e02bf9a0f22a2f383bff7 (diff)
downloadbuilder-db5448138b313407d8062c04ed6df458ff64c66f.tar.xz
lib/mysql-functions: solve a few todos in mysql_generate_package_metadata() - we now have versioned install targets!
-rwxr-xr-xlib/mysql-functions524
1 files changed, 216 insertions, 308 deletions
diff --git a/lib/mysql-functions b/lib/mysql-functions
index 6c8154c..40843e3 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -210,6 +210,8 @@ mysql_add_package_source() {
mysql_generate_package_metadata() {
+ mysql_load_min_and_max_versions
+
( # new shell is intentional
current_repository_id="$1"
package="$2"
@@ -260,9 +262,6 @@ mysql_generate_package_metadata() {
mysql_add_package_source "${pkgbase}" "${git_revision}" "${old_git_revision}" "${mod_git_revision}" "${repository}" "${temp_dir}/SRCINFO"
)
- # TODO: architectures should be identified by id, not "(SELECT ...)"
-
- # add the build assignment(s)
{
sed -n '
s/^\tarch = //
@@ -272,342 +271,251 @@ mysql_generate_package_metadata() {
sort -u | \
grep -vxF 'any' || \
echo 'any'
- } | \
- base64_encode_each | \
- while read -r arch; do
- printf 'INSERT INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
- printf ' SELECT %s,`architectures`.`id`,NULL,0,0' \
- "${package_source_id}"
- printf ' FROM `architectures`'
- printf ' WHERE `architectures`.`name` = from_base64("%s")' \
- "${arch}"
- printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`build_assignments`.`id`);\n'
- printf 'SELECT "%s",LAST_INSERT_ID();\n' \
- "${arch}"
- done | \
- mysql_run_query > \
- "${temp_dir}/build-assignments.id"
+ } > \
+ "${temp_dir}/architectures"
# select any specific arch (which will be building the 'any' part of a split package)
any_arch=$(
- cut -f1 < \
- "${temp_dir}/build-assignments.id" | \
- sort | \
- head -n1 | \
- tr -d '\n' | \
- base64 -w0
+ tail -n1 "${temp_dir}/architectures"
)
# iterate over all pkgnames
grep '^pkgname = ' "${temp_dir}/SRCINFO" | \
cut -d' ' -f3 | \
while read -r pkgname; do
- pkgname64=$(
- printf '%s' "${pkgname}" | \
- base64 -w0
- )
sed -n '
/^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/,/^$/ {
+ s/^pkgname/\t\0/
/^\S/d
+ /^\s*$/d
s/^\s*//
p
}
' "${temp_dir}/SRCINFO" > \
- "${temp_dir}/BINARYINFO.${pkgname64}"
+ "${temp_dir}/SRCINFO.tmp"
- # iterate over all archs
- grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \
- cut -d' ' -f3 | \
- while read -r arch; do
- arch64=$(
- printf '%s' "${arch}" | \
- base64 -w0
- )
- {
- {
- # this binary package is either built by the build_assignment
- # with the identical arch (if existent) or by the one
- # with arch=$any_arch
- grep "^${arch64}"'\s' "${temp_dir}/build-assignments.id" || \
- grep "^${any_arch}"'\s' "${temp_dir}/build-assignments.id"
- } | \
- sed '
- s/^.*\s/build-assignment-id = /
- '
- sed '
- s/^\(\S\+\)_'"${arch}"' = /\1 = /
- ' "${temp_dir}/BINARYINFO.${pkgname64}"
- } > \
- "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}"
- done
- done
-
- # TODO: Data should be loaded by "LOAD DATA LOCAL INFILE ...". This
- # has three advantages: 1st, we separate data from the commands
- # (aesthetic), 2nd, we can prepare/check the data to load "offline",
- # 3rd, we are less sensitive to errors in the middle of the commit.
- # For this we need some intermediate files which are loaded into
- # temporary tables:
- # - pkgbase -> build_assignment (multiple archs possible!)
- # - build_assignment -> pkgname (split packages possible!)
- # - pkgname -> dependency
- # - pkgname -> provides
-
- # now each "ARCHINFO $pkgname $arch" file represents one binary package
- find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \
- while read -r _ pkgname arch; do
- build_assignment_id=$(
- grep '^build-assignment-id = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3
- )
- pkgver=$(
- grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- tr -d '\n' | \
- base64 -w0
- )
- pkgrel=$(
- grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/\.[0-9]\+$//' | \
- tr -d '\n' | \
- base64 -w0
- )
epoch=$(
{
- grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \
+ grep -m1 '^epoch = ' "${temp_dir}/SRCINFO.tmp" || \
echo 'epoch = 0'
} | \
- cut -d' ' -f3 | \
- tr -d '\n' | \
- base64 -w0
- )
- # TODO: allow versioned install_targets
- provides=$(
- grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- makedepends=$(
- grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
- )
- checkdepends=$(
- grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
+ cut -d' ' -f3
)
- rundepends=$(
- grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
- cut -d' ' -f3 | \
- sed 's/[<>=].*$//' | \
- base64_encode_each
+ pkgver=$(
+ grep -m1 '^pkgver = ' "${temp_dir}/SRCINFO.tmp" | \
+ cut -d' ' -f3
)
- sub_pkgrel=$(
- printf '(SELECT COALESCE('
- # do not add binary packages which are currently on the
- # build-list or in $current_repository_id (beware of split
- # packages!)
- printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
- mysql_join_binary_packages_architectures
- mysql_join_binary_packages_binary_packages_in_repositories
- printf ' WHERE'
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- printf ' `architectures`.`name`=from_base64("%s")' \
- "${arch}"
- printf ' AND `binary_packages_in_repositories`.`repository` IN (%s,%s)),' \
- "${repository_ids__any_build_list}" \
- "${current_repository_id}"
- # max(sub_pkgrel)+1
- printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
- mysql_join_binary_packages_architectures
- printf ' WHERE'
- printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then
- # 'any' gets higher sub_pkgrel than any architecture
- printf ' 1'
- else
- # not-'any' gets higher sub_pkgrel than same or 'any' architecture
- printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \
- "${arch}"
- fi
- printf ')'
- printf ',0))'
+ pkgrel=$(
+ grep -m1 '^pkgrel = ' "${temp_dir}/SRCINFO.tmp" | \
+ cut -d' ' -f3
)
- { # TODO: identify arch by id
- printf 'INSERT IGNORE INTO `binary_packages` ('
- printf '`%s`,' \
- 'build_assignment' \
- 'architecture' \
- 'epoch' \
- 'pkgver' \
- 'pkgrel' \
- 'pkgname' \
- 'sub_pkgrel' \
- 'has_issues' \
- 'is_tested'
- printf ') SELECT '
- printf '%s,' \
- "${build_assignment_id}"
- printf '`%s`.`id`,' \
- 'architectures'
- printf 'from_base64("%s"),' \
- "${epoch}" \
- "${pkgver}" \
- "${pkgrel}" \
- "${pkgname}"
- printf '%s,0,0 FROM `architectures`' \
- "${sub_pkgrel}"
- printf ' WHERE'
- printf ' `architectures`.`name` = from_base64("%s")' \
- "${arch}"
- printf ';\n'
- printf 'INSERT IGNORE INTO `binary_packages_in_repositories` (`package`,`repository`,`is_to_be_deleted`)'
- printf ' SELECT LAST_INSERT_ID(),%s,0;\n' \
- "${current_repository_id}"
- printf 'COMMIT;\n'
- } | \
- sed '
- s|,)|)|g
- s| JOIN WHERE | WHERE |
- s| AND;$|;|
- ' >> \
- "${temp_dir}/add-binary-packages-command"
- {
- printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \
- 'provides' \
- 'makedepends' \
- 'checkdepends' \
- 'rundepends'
-
- printf 'INSERT INTO `provides` VALUES\n'
- echo "${provides}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"),"-",from_base64("%s"))),\n' \
- "${pkgname}" \
- "${epoch}" \
- "${pkgver}" \
- "${pkgrel}"
- printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"))),\n' \
- "${pkgname}" \
- "${epoch}" \
- "${pkgver}"
- printf ' (from_base64("%s"));\n' \
- "${pkgname}"
-
- printf 'INSERT INTO `rundepends` VALUES\n'
- echo "${rundepends}" | \
- sort -u | \
- grep -vxF '' | \
- sed '
- s|^| (from_base64("|
- s|$|")),|
- '
- printf ' ("base");\n'
- echo "${checkdepends}" | \
- sort -u | \
- grep -vxF '' | \
+ # iterate over all archs
+ grep '^arch = ' "${temp_dir}/SRCINFO.tmp" | \
+ cut -d' ' -f3 | \
+ while read -r arch; do
+ build_arch=$(
+ # this binary package is either built by the build_assignment
+ # with the identical arch (if existent) or by the one
+ # with arch=$any_arch
+ grep -xF "${arch}" "${temp_dir}/architectures" || \
+ echo "${any_arch}"
+ )
sed '
- s|^| (from_base64("|
- s|$|")),|
- 1 s/^/INSERT INTO `checkdepends` VALUES \n/
- $ s/,$/;/
- '
-
- printf 'INSERT INTO `makedepends` VALUES\n'
- echo "${makedepends}" | \
- sort -u | \
- grep -vxF '' | \
+ s/^\(\S\+\)_'"${arch}"' = /\1 = /
+ s/^pkgname = \(\S\+\)$/\0\nprovides = \1='"${epoch}"':'"${pkgver}"'\ndepends = base\nmakedepends = base-devel/
+ ' "${temp_dir}/SRCINFO.tmp" | \
+ sed '
+ s/^arch = \S\+$/arch/
+ t
+ s/^\(groups\|provides\) = /provides\t/
+ t mangle_version
+ s/^makedepends = /makedepends\t/
+ t mangle_version
+ s/^checkdepends = /checkdepends\t/
+ t mangle_version
+ s/^depends = /rundepends\t/
+ t mangle_version
+ d
+ :mangle_version
+ s/\(=\|<\|<=\|>=\|>\)\([^[:space:]-]\+\)$/\t\1\t\2/
+ t split_epoch
+ h
+ s/$/\t>=\t'"${min_version}"'/
+ s/\(\s[0-9]\+\):\(\S\+\)$/\1\t\2/
+ p
+ g
+ s/$/\t<=\t'"${max_version}"'/
+ :split_epoch
+ s/\(\s[0-9]\+\):\(\S\+\)$/\1\t\2/
+ t
+ s/\s\S\+$/\t0\0/
+ ' | \
sed '
- s|^| (from_base64("|
- s|$|")),|
+ s/^/'"$(
+ printf '%s\t' \
+ "${pkgname}" \
+ "${arch}" \
+ "${epoch}" \
+ "${pkgver}" \
+ "${pkgrel}" \
+ "${build_arch}"
+ )"'/
'
- printf ' ("base-devel");\n'
-
- printf 'INSERT IGNORE INTO `install_targets` (`name`)'
- printf ' SELECT (`name`) FROM `%s` UNION' \
- 'provides' \
- 'makedepends' \
- 'checkdepends' \
- 'rundepends' | \
- sed 's| UNION$|;\n|'
-
- for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
- case "${link}" in
- 'provides')
- printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT'
- printf ' `binary_packages`.`id`,`install_targets`.`id` FROM'
- ;;
- 'makedepends'|'checkdepends'|'rundepends')
- printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT'
- printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM'
- printf ' `dependency_types` JOIN'
- ;;
- esac
- printf ' `binary_packages`'
- mysql_join_binary_packages_architectures
- printf ' JOIN `install_targets`'
- printf ' JOIN `%s`' "${link}"
- printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
- printf ' WHERE'
- if [ "${link}" = 'makedepends' ] || \
- [ "${link}" = 'checkdepends' ] || \
- [ "${link}" = 'rundepends' ]; then
- printf ' `dependency_types`.`name` = "%s" AND' \
- "${link%depends}"
- fi
- # TODO: identify binary_packages by id
- printf ' `binary_packages`.`%s` = from_base64("%s") AND' \
- 'epoch' "${epoch}" \
- 'pkgver' "${pkgver}" \
- 'pkgrel' "${pkgrel}" \
- 'pkgname' "${pkgname}"
- # we do not want to match the sub_pkgrel:
- # a) it is tedious to do so (because it may be calculated
- # dynamically)
- # b) it is not necessary to do so: if only the sub_pkgrel
- # changed, the dependencies and provided install_targets
- # should not have changed
- printf ' `architectures`.`name` = from_base64("%s");\n' \
- "${arch}"
- # the repository is of no relevance: it hardly matters for
- # the dependencies
done
-
- printf 'DROP TABLE `%s`;\n' \
- 'provides' \
- 'makedepends' \
- 'checkdepends' \
- 'rundepends'
- } >> \
- "${temp_dir}/add-install-targets-command"
- done
+ rm "${temp_dir}/SRCINFO.tmp"
+ done | \
+ sort -u > \
+ "${temp_dir}/database-input"
{
- if [ -s "${temp_dir}/add-binary-packages-command" ]; then
- cat "${temp_dir}/add-binary-packages-command"
- fi
- if [ -s "${temp_dir}/add-install-targets-command" ]; then
- cat "${temp_dir}/add-install-targets-command"
- fi
+ printf 'CREATE TEMPORARY TABLE `links`('
+ printf '`pkgname` VARCHAR(64),'
+ printf '`architecture` VARCHAR(16),'
+ printf '`epoch` MEDIUMINT,'
+ printf '`pkgver` VARCHAR(64),'
+ printf '`pkgrel` MEDIUMINT,'
+ printf '`build_architecture` VARCHAR(16),'
+ printf '`type` VARCHAR(16),'
+ printf '`install_target_name` VARCHAR(128),'
+ printf '`version_relation` VARCHAR(2),'
+ printf '`version_epoch` MEDIUMINT,'
+ printf '`version_version` VARCHAR(64)'
+ printf ');\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `links`;\n' \
+ "${temp_dir}/database-input"
+
+ printf 'SET @sub_pkgrel=('
+ printf 'SELECT MAX(`binary_packages`.`sub_pkgrel`)'
+ printf ' FROM `links`'
+ printf ' JOIN `architectures`'
+ printf ' ON `architectures`.`name`=`links`.`architecture`'
+ mysql_join_architectures_binary_packages
+ printf ' AND `binary_packages`.`%s`=`links`.`%s`' \
+ 'pkgname' 'pkgname' \
+ 'epoch' 'epoch' \
+ 'pkgver' 'pkgver' \
+ 'pkgrel' 'pkgrel'
+ printf ');\n'
+
+ printf 'INSERT IGNORE INTO `build_assignments` ('
+ printf '`package_source`,'
+ printf '`architecture`'
+ printf ') SELECT'
+ printf ' %s,`architectures`.`id`' \
+ "${package_source_id}"
+ printf ' FROM `links`'
+ printf ' JOIN `architectures`'
+ printf ' ON `architectures`.`name`=`links`.`build_architecture`'
+ printf ' WHERE `links`.`type`=`arch`;\n'
+
+ printf 'INSERT IGNORE INTO `binary_packages` ('
+ printf '`build_assignment`,'
+ printf '`pkgname`,'
+ printf '`epoch`,'
+ printf '`pkgver`,'
+ printf '`pkgrel`,'
+ printf '`sub_pkgrel`,'
+ printf '`architecture`'
+ printf ') SELECT'
+ printf ' `build_assignments`.`id`,'
+ printf '`links`.`%s`,' \
+ 'pkgname' \
+ 'epoch' \
+ 'pkgver' \
+ 'pkgrel'
+ printf '@sub_pkgrel,'
+ printf '`architectures`.`id`'
+ printf ' FROM `links`'
+ printf ' JOIN `architectures`'
+ printf ' ON `links`.`architecture`=`architectures`.`name`'
+ printf ' JOIN `architectures` AS `ba_a`'
+ printf ' ON `ba_a`.`name`=`links`.`build_architecture`'
+ mysql_join_architectures_build_assignments 'ba_a'
+ printf ' WHERE `links`.`type`=`arch`;\n'
+
+ printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
+ printf ' `package`'
+ printf ' `repository`'
+ printf ') SELECT'
+ printf ' `binary_packages`.`id`,'
+ printf '%s' \
+ "${current_repository_id}"
+ printf ' FROM `links`'
+ printf ' JOIN `architectures`'
+ printf ' ON `architectures`.`name`=`links`.`architecture`'
+ mysql_join_architectures_binary_packages
+ printf ' AND `binary_packages`.`sub_pkgrel`=@sub_pkgrel'
+ printf ' AND `binary_packages`.`%s`=`links`.`%s`' \
+ 'pkgname' 'pkgname' \
+ 'epoch' 'epoch' \
+ 'pkgver' 'pkgver' \
+ 'pkgrel' 'pkgrel'
+ printf ';\n'
+
+ printf 'INSERT IGNORE INTO `install_targets` (`name`)'
+ printf ' SELECT `links`.`install_target_name`'
+ printf ' FROM `links`;\n'
+
+ printf 'INSERT IGNORE INTO `versions` (`epoch`,`version`)'
+ printf ' SELECT'
+ printf ' `links`.`version_epoch`,'
+ printf '`links`.`version_version`'
+ printf ' FROM `links`;\n'
+
+ for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
+ case "${link}" in
+ 'provides')
+ printf 'INSERT IGNORE INTO `install_target_providers` ('
+ printf '`package`,'
+ printf '`install_target`,'
+ printf '`version`'
+ printf ') SELECT'
+ printf ' `binary_packages`.`id`,'
+ printf '`install_targets`.`id`,'
+ printf '`versions`.`id`'
+ ;;
+ 'makedepends'|'checkdepends'|'rundepends')
+ printf 'INSERT IGNORE INTO `dependencies` ('
+ printf '`dependent`,'
+ printf '`depending_on`,'
+ printf '`dependency_type`,'
+ printf '`version`,'
+ printf '`version_relation`,'
+ printf ') SELECT'
+ printf ' `binary_packages`.`id`,'
+ printf '`install_targets`.`id`,'
+ printf '`dependency_types`.`id`,'
+ printf '`versions`.`id`,'
+ printf '`links`.`version_relation`,'
+ ;;
+ esac
+ printf ' FROM `links`'
+ printf ' JOIN `architectures`'
+ printf ' ON `architectures`.`name`=`links`.`architecture`'
+ mysql_join_architectures_binary_packages
+ printf ' AND `binary_packages`.`sub_pkgrel`=@sub_pkgrel'
+ printf ' AND `binary_packages`.`%s`=`links`.`%s`' \
+ 'pkgname' 'pkgname' \
+ 'epoch' 'epoch' \
+ 'pkgver' 'pkgver' \
+ 'pkgrel' 'pkgrel'
+ printf ' JOIN `install_targets`'
+ printf ' ON `install_targets`.`name`=`links`.`install_target_name`'
+ printf ' JOIN `versions`'
+ printf ' ON `versions`.`epoch`=`links`.`install_target_epoch`'
+ printf ' AND `versions`.`version`=`links`.`install_target_version`'
+ if [ "${link}" = 'makedepends' ] || \
+ [ "${link}" = 'checkdepends' ] || \
+ [ "${link}" = 'rundepends' ]; then
+ printf ' JOIN `dependency_types`'
+ printf ' ON `dependency_types`.`name`="%s"' \
+ "${link%depends}"
+ fi
+ printf ' WHERE `links`.`type`="%s";\n' \
+ "${link}"
+ done
+ printf 'DROP TEMPORARY TABLE `links`;\n'
} | \
mysql_run_query