diff options
Diffstat (limited to 'lib')
| -rwxr-xr-x | lib/common-functions | 75 | ||||
| -rwxr-xr-x | lib/mysql-functions | 350 |
2 files changed, 108 insertions, 317 deletions
diff --git a/lib/common-functions b/lib/common-functions index ef10367..7093c12 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -549,8 +549,7 @@ recursively_umount_and_rm() { # mangle_pkgbuild $PKGBUILD [$sub_pkgrel] # mangle $arch in PKBUILDs to contain i486, i586, i686 # append $sub_pkgrel to the pkgrel - -# TODO: actually include the other architectures +# remove "lib32-" from {make,check,opt,}depends mangle_pkgbuild() { local PKGBUILD="$1" @@ -568,53 +567,20 @@ mangle_pkgbuild() { sed -i ' /^arch=[^#]*any/!{ - /^arch=(/s/(/(i686 / + /^arch=(/s/(/(i686 i486 pentium3 / } s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ ' "${PKGBUILD}" -} -# find_newest_of_git_revisions -# find newest git revision of the ones provided at stdin -# (assuming linear history) - -find_newest_of_git_revisions() { - local revisions - local repo - revisions=$(cat) - - if [ "$( - echo "${revisions}" | \ - wc -l - )" -eq 1 ]; then - - echo "${revisions}" - return - - fi - - repo=$( - find_repository_with_commit \ - "$( - echo "${revisions}" | \ - grep -xm1 '[0-9a-f]\{40\}' - )" - ) - - eval 'repo="${repo_paths__'"${repo}"'}"' - - echo "${revisions}" | \ - xargs -rn1 git -C "${repo}" rev-parse | \ - { - newest='' - while read -r current; do - if [ -z "${newest}" ] || \ - git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then - newest="${current}" - fi - done - echo "${newest}" + sed -i ' + /^\s*\(\(make\|check\|opt\|\)depends\)=(/ { + :a + s,^\s*\(\S[^=]*\)=(\([^()]*\(#[^\n]*\n\)\?\)*),\0\n\1=(${\1[\@]/lib32-/})\n, + t + $!N + ba } + ' "${PKGBUILD}" } # find_package_repository_to_package $package $git_repository $git_commit @@ -921,10 +887,23 @@ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { fi } -# irc_say -# say content of stdin in irc channel +# irc_say $channel [copy] +# say content of stdin in irc channel $channel (default: #archlinux32) +# and print copy to stdout if 'copy' is given + +# shellcheck disable=SC2120 irc_say() { - if [ -p "${irc_dir}/#archlinux32/in" ]; then - sponge "${irc_dir}/#archlinux32/in" + local channel + if [ -z "$1" ]; then + channel='#archlinux32' + else + channel="$1" + fi + if [ -p "${irc_dir}/${channel}/in" ]; then + if [ "$2" = 'copy' ]; then + pee cat 'sponge '"${irc_dir}/${channel}/in" + else + sponge "${irc_dir}/${channel}/in" + fi fi } diff --git a/lib/mysql-functions b/lib/mysql-functions index 83b9296..71097cb 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -2,6 +2,9 @@ # contains functions used to access mysql db +# TODO: return ids on INSERT/UPDATE queries and use those in subsequent +# queries + # shellcheck disable=SC2016,SC2039 if [ -z "${base_dir}" ]; then @@ -64,6 +67,7 @@ mysql_run_query() { >&2 printf 'I could not complete a mysql query!\n' if [ ! -s "${work_dir}/build-master-sanity" ] && \ [ -z "${file_name_extra}" ]; then + # shellcheck disable=SC2119 printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_stdin##*/}" \ | irc_say @@ -122,10 +126,8 @@ mysql_add_package_source() { else uses_modification=0 fi - printf 'A' >&2 if grep -qx 'PKGBUILD = \S\+' "${srcinfo_file}"; then uses_upstream=1 - printf 'B' >&2 eval 'repo_path="${repo_paths__'"$(sed -n 's/^upstream_git_repository = //;T;p' "${srcinfo_file}")"'}"' pkgbuild=$( sed -n ' @@ -134,26 +136,19 @@ mysql_add_package_source() { p ' "${srcinfo_file}" ) - printf 'C' >&2 commit_time=$( git -C "${repo_path}" log -n 1 --pretty=format:%ct "${old_git_revision}${git_revision}" -- "${pkgbuild}" ) - printf 'D' >&2 if [ -z "${commit_time}" ]; then # We may have modified git_mod_revision! - printf 'E' >&2 commit_time=$( git -C "${repo_path}" log -n 1 --pretty=format:%ct "${git_revision}" -- "${pkgbuild}" ) - printf 'F' >&2 fi else - printf 'G' >&2 uses_upstream=0 commit_time=$(date '+%s') - printf 'H' >&2 fi - printf 'I' >&2 upstream_package_repository_id=$( { @@ -176,7 +171,7 @@ mysql_add_package_source() { fi { - printf 'INSERT IGNORE INTO package_sources' + printf 'INSERT INTO `package_sources`' printf ' (`pkgbase`,`git_revision`,`mod_git_revision`,`upstream_package_repository`,`uses_upstream`,`uses_modification`,`commit_time`)' printf ' VALUES (' printf 'from_base64("%s"),' \ @@ -196,60 +191,12 @@ mysql_add_package_source() { "${upstream_package_repository_id}" \ "${uses_upstream}" \ "${uses_modification}" - printf 'from_unixtime(%s));\n' \ + printf 'from_unixtime(%s))' \ "${commit_time}" + printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`package_sources`.`id`);\n' + printf 'SELECT LAST_INSERT_ID();\n' } | \ mysql_run_query - printf 'J' >&2 -} - -# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel - -# shellcheck disable=SC2031,SC2086,SC2154 -mysql_show_binary_package() { - local names='pkgname pkgver pkgrel sub_pkgrel' - local name - for name in ${names}; do - eval 'local '"${name}" - eval "${name}"'=$( - printf "%s" "$1" | - base64 -w0 - )' - shift - done - - { - printf 'SELECT' - printf ' `%s`.`%s`,' \ - 'repositories' 'name' \ - 'binary_packages' 'pkgname' \ - 'package_sources' 'pkgver' \ - 'package_sources' 'pkgrel' \ - 'binary_packages' 'sub_pkgrel' \ - 'architectures' 'name' \ - 'package_sources' 'pkgbase' \ - 'package_sources' 'git_revision' \ - 'package_sources' 'mod_git_revision' \ - 'upstream_repositories' 'name' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_architectures - mysql_join_binary_packages_repositories - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' WHERE' - printf ' `%s`.`%s` = from_base64("%s") AND' \ - 'binary_packages' 'pkgname' "${pkgname}" \ - 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \ - 'package_sources' 'pkgver' "${pkgver}" \ - 'package_sources' 'pkgrel' "${pkgrel}" - printf ';' - } | \ - sed ' - s|, FROM| FROM|g - s|AND;|;|g - ' | \ - mysql_run_query --html --column-names } # mysql_generate_package_metadata $current_repository $package $git_revision $old_git_revision $mod_git_revision $repository @@ -286,7 +233,6 @@ mysql_generate_package_metadata() { temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - printf '.a' >&2 if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" exit 2 @@ -306,7 +252,6 @@ mysql_generate_package_metadata() { fi printf '\n\n' >> "${temp_dir}/SRCINFO" - printf '.b' >&2 pkgbase=$( grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \ cut -d' ' -f3 @@ -315,86 +260,55 @@ mysql_generate_package_metadata() { >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n' exit 2 fi - printf '.c' >&2 # add the package source - mysql_add_package_source "${pkgbase}" "${git_revision}" "${old_git_revision}" "${mod_git_revision}" "${repository}" "${temp_dir}/SRCINFO" - printf '.d' >&2 + package_source_id=$( + mysql_add_package_source "${pkgbase}" "${git_revision}" "${old_git_revision}" "${mod_git_revision}" "${repository}" "${temp_dir}/SRCINFO" + ) # now we encode everything in base64 current_repository=$( printf '%s' "${current_repository}" | \ base64 -w0 ) - pkgbase=$( - printf '%s' "${pkgbase}" | \ - base64 -w0 - ) - git_revision=$( - printf '%s' "${git_revision}" | \ - base64 -w0 - ) - mod_git_revision=$( - printf '%s' "${mod_git_revision}" | \ - base64 -w0 - ) - repository=$( - printf '%s' "${repository}" | \ - base64 -w0 - ) # add the build assignment(s) { - archs=$( - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - grep -vxF 'any' | \ - sort -u - ) - if [ -z "${archs}" ]; then + sed -n ' + s/^\tarch = // + T + p + ' "${temp_dir}/SRCINFO" | \ + sort -u | \ + grep -vxF 'any' || \ echo 'any' - else - printf '%s\n' "${archs}" - fi } | \ + base64_encode_each | \ while read -r arch; do - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0' - printf ' FROM `architectures` JOIN `package_sources`' + printf 'INSERT INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT %s,`architectures`.`id`,NULL,0,0' \ + "${package_source_id}" + printf ' FROM `architectures`' printf ' WHERE `architectures`.`name` = from_base64("%s")' \ - "$( - printf '%s' "${arch}" | \ - base64 -w0 - )" - printf ' AND `package_sources`.`%s` = from_base64("%s")' \ - 'pkgbase' "${pkgbase}" \ - 'git_revision' "${git_revision}" \ - 'mod_git_revision' "${mod_git_revision}" - printf ';\n' - done > \ - "${temp_dir}/add-build-assignments-command" - - # TODO: correctly link between binary_packages and build_assignments using any_arch + "${arch}" + printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`build_assignments`.`id`);\n' + printf 'SELECT "%s",LAST_INSERT_ID();\n' \ + "${arch}" + done | \ + mysql_run_query > \ + "${temp_dir}/build-assignments.id" - # shellcheck disable=SC2034 # select any specific arch (which will be building the 'any' part of a split package) any_arch=$( - { - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - sort -r | \ - grep -vxFm 1 'any' || \ - echo 'any' - } | \ - base64_encode_each + cut -f1 < \ + "${temp_dir}/build-assignments.id" | \ + sort | \ + head -n1 | \ + tr -d '\n' | \ + base64 -w0 ) + # iterate over all pkgnames grep '^pkgname = ' "${temp_dir}/SRCINFO" | \ cut -d' ' -f3 | \ while read -r pkgname; do @@ -403,22 +317,15 @@ mysql_generate_package_metadata() { base64 -w0 ) sed -n ' - /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{ - :a - N - /\n$/{ - p - T - } - ba - } - ' "${temp_dir}/SRCINFO" | \ - sed ' + /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/,/^$/ { /^\S/d s/^\s*// - ' > \ + p + } + ' "${temp_dir}/SRCINFO" > \ "${temp_dir}/BINARYINFO.${pkgname64}" + # iterate over all archs grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \ cut -d' ' -f3 | \ while read -r arch; do @@ -426,24 +333,51 @@ mysql_generate_package_metadata() { printf '%s' "${arch}" | \ base64 -w0 ) - sed ' - s/^\(\S\+\)_'"${arch}"' = /\1 = / - ' "${temp_dir}/BINARYINFO.${pkgname64}" > \ + { + { + # this binary package is either built by the build_assignment + # with the identical arch (if existent) or by the one + # with arch=$any_arch + grep "^${arch64}"'\s' "${temp_dir}/build-assignments.id" || \ + grep "^${any_arch}"'\s' "${temp_dir}/build-assignments.id" + } | \ + sed ' + s/^.*\s/build-assignment-id = / + ' + sed ' + s/^\(\S\+\)_'"${arch}"' = /\1 = / + ' "${temp_dir}/BINARYINFO.${pkgname64}" + } > \ "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}" done done + + # TODO: This should become multiple mysql queries: + # 1st inserting install_targets + # 2nd inserting binary_packages (and binary_packages_in_repositories) + # 3rd,4th inserting dependencies and install_target_providers + # The links between the above should _solely_ be made by the ids of + # the respective rows. + + # now each "ARCHINFO $pkgname $arch" file represents one binary package find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \ while read -r _ pkgname arch; do + build_assignment_id=$( + grep '^build-assignment-id = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 + ) pkgver=$( grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ - base64_encode_each + tr -d '\n' | \ + base64 -w0 ) pkgrel=$( grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ sed 's/\.[0-9]\+$//' | \ - base64_encode_each + tr -d '\n' | \ + base64 -w0 ) epoch=$( { @@ -451,8 +385,10 @@ mysql_generate_package_metadata() { echo 'epoch = 0' } | \ cut -d' ' -f3 | \ - base64_encode_each + tr -d '\n' | \ + base64 -w0 ) + # TODO: allow versioned install_targets provides=$( grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ @@ -534,8 +470,8 @@ mysql_generate_package_metadata() { 'is_tested' \ 'is_to_be_deleted' printf ') SELECT ' + printf '%s,' "${build_assignment_id}" printf '`%s`.`id`,' \ - 'build_assignments' \ 'repositories' \ 'architectures' printf 'from_base64("%s"),' \ @@ -547,23 +483,16 @@ mysql_generate_package_metadata() { "${sub_pkgrel}" printf ' `%s` JOIN' \ 'repositories' \ - 'architectures' \ - 'build_assignments' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories + 'architectures' printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'repositories' 'name' "${current_repository}" \ - 'architectures' 'name' "${arch}" \ - 'package_sources' 'pkgbase' "${pkgbase}" \ - 'package_sources' 'git_revision' "${git_revision}" \ - 'package_sources' 'mod_git_revision' "${mod_git_revision}" \ - 'upstream_repositories' 'name' "${repository}" + 'architectures' 'name' "${arch}" printf ';\n' } | \ sed ' s|,)|)|g - s| JOIN JOIN | JOIN | + s| JOIN WHERE | WHERE | s| AND;$|;| ' >> \ "${temp_dir}/add-binary-packages-command" @@ -689,12 +618,8 @@ mysql_generate_package_metadata() { } >> \ "${temp_dir}/add-install-targets-command" done - printf '.e' >&2 { - if [ -s "${temp_dir}/add-build-assignments-command" ]; then - cat "${temp_dir}/add-build-assignments-command" - fi if [ -s "${temp_dir}/add-binary-packages-command" ]; then cat "${temp_dir}/add-binary-packages-command" fi @@ -703,7 +628,6 @@ mysql_generate_package_metadata() { fi } | \ mysql_run_query - printf '.f' >&2 ) } @@ -999,118 +923,6 @@ mysql_query_select_pkgbase_and_revision() { mysql_join_package_sources_upstream_repositories } -# mysql_repair_binary_packages_without_build_assignment -# try to generate valid build assignments to binary packages without -# a valid one yet -mysql_repair_binary_packages_without_build_assignment() { - { - printf 'SELECT ' - printf '`binary_packages`.`id`' - printf ',replace(to_base64(%s),"\\n","")' \ - '`binary_packages`.`pkgname`' \ - '`architectures`.`name`' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_architectures - printf ' WHERE `binary_packages`.`build_assignment`<0' - } | \ - mysql_run_query | \ - while read -r id pkgname arch; do - pkgname=$( - printf '%s' "${pkgname}" | \ - base64 -d - ) - pkgbase=$( - curl -Ss "$( - printf 'https://www.archlinux.org/packages/search/json/?name=%s' \ - "${pkgname}" - )" | \ - sed ' - s/^.*"results":\s*\[// - s/}\s*,\s*{/\n/g - ' | \ - grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \ - tr ',' '\n' | \ - grep '"pkgbase":' | \ - cut -d'"' -f4 | \ - sort -u | \ - head -n1 - ) - if [ -z "${pkgbase}" ] && \ - { - printf 'SELECT count(*) FROM `package_sources`' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ - "$(printf '%s' "${pkgname}" | base64 -w0)" - } | \ - mysql_run_query | \ - grep -qvxF '0'; then - pkgbase="${pkgname}" - fi - if [ -z "${pkgbase}" ]; then - >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}" - continue - fi - pkgbase=$( - printf '%s' "${pkgbase}" | \ - base64 -w0 - ) - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0' - printf ' FROM `package_sources`' - printf ' JOIN `architectures`' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}" - printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}" - printf ' LIMIT 1;\n' - printf 'UPDATE `binary_packages`' - printf ' JOIN `build_assignments`' - mysql_join_binary_packages_build_assignments - printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' WHERE `binary_packages`.`id`=%s' "${id}" - printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" - done | \ - mysql_run_query -} - -# mysql_remove_duplicate_binary_packages -# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel, -# having difference of 1 in sub_pkgrel - -mysql_remove_duplicate_build_order() { - { - printf 'CREATE TEMPORARY TABLE `ren`' - printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n' - printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)' - printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`' - printf ' FROM `binary_packages` as `old`' - printf ' JOIN `binary_packages` as `new` ON' - printf ' `old`.`%s`=`new`.`%s` AND' \ - 'pkgname' 'pkgname' \ - 'epoch' 'epoch' \ - 'pkgver' 'pkgver' \ - 'pkgrel' 'pkgrel' - printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' - mysql_join_binary_packages_repositories 'old' 'orep' - mysql_join_binary_packages_repositories 'new' 'nrep' - printf ' WHERE `orep`.`name`!="to-be-decided"' - printf ' AND `nrep`.`name`="to-be-decided";\n' - printf 'UPDATE IGNORE `dependencies`' - printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`' - printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n' - printf 'UPDATE IGNORE `install_target_providers`' - printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`' - printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n' - printf 'DELETE FROM `binary_packages`' - printf ' WHERE EXISTS (' - printf 'SELECT * FROM `ren`' - printf ' WHERE `ren`.`old`=`binary_packages`.`id`' - printf ');\n' - printf 'UPDATE IGNORE `binary_packages`' - printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`' - printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' - printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' - } | \ - mysql_run_query -} - # mysql_package_name_query # print a mysql query of the full name of a package file |