path: root/bin/build-packages

#!/bin/sh

# build packages one by one, then upload the binary package to the repository server

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# TODO: build other 'architectures', too (pentium4, i486)

# TODO: report back memory and hdd stats to the build master on successful build

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'build-packages: build package(s) on the build-list'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -h|--help:   Show this help and exit.'
  >&2 echo '  -l|--local pkgname.git-revision.git-mod-revision.repository:'
  >&2 echo '               Build the given package without asking / reporting to the'
  >&2 echo '               build master. Cannot be combined with -n, -p, -t or -x.'
  >&2 echo '             where:'
  >&2 echo '               - git-revision: packages-HEAD or community-HEAD or a valid commit'
  >&2 echo '               - git-mod-revision: work-tree or a valid commit'
  >&2 echo '             example:'
  >&2 echo '               build-package -l   which.packages-HEAD.work-tree.core'
  >&2 echo '  -n count:    Build $count packages (if available), then exit.'
  >&2 echo '               $count=0 is interpreted as infinity.'
  >&2 echo '               The default is $count=1 or 0 iff -t or -x is given.'
  >&2 echo '               Cannot be combined with -l.'
  >&2 echo '  -p|--prefer pkgname:'
  >&2 echo '               Ask the build master to get an assignment for the given'
  >&2 echo '               package but built what is offered anyway. Cannot be combined'
  >&2 echo '               with -l.'
  >&2 echo '  -s|--straw $straw:'
  >&2 echo '               Use this straw instead of the preconfigured ones. -- May be'
  >&2 echo '               given multiple times to allow using multiple straws.'
  >&2 echo '  -t seconds:  Do not request new assignment(s) $seconds seconds after start.'
  >&2 echo '               Cannot be combined with -l.'
  >&2 echo '  -x:          If package build fails, do not request new assignment(s).'
  >&2 echo '               Cannot be combined with -l.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o hl:n:p:s:t:ux \
    --long help \
    --long local: \
    --long prefer: \
    --long straw: \
    --long upload \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

unset count
unset forced_package
unset forced_straws
unset prefered_package
exit_after_failure=false
timeout=0

while true
do
  case "$1" in
    -h|--help)
      usage 0
    ;;
    -l|--local)
      shift
      if [ -n "${forced_package}" ]; then
        >&2 echo 'Option -l, --local can be given only once.'
        usage
      fi
      forced_package="$1"
    ;;
    -n)
      shift
      count="$1"
      [ "${count}" -eq 0 ] && \
        count=-1
    ;;
    -p|--prefer)
      shift
      if [ -n "${prefered_package}" ]; then
        >&2 echo 'Option -p, --prefer can be given only once.'
        usage
      fi
      prefered_package="$1"
    ;;
    -s|--straw)
      shift
      forced_straws="${forced_straws} $1"
    ;;
    -t)
      shift
      timeout="$1"
    ;;
    -x)
      exit_after_failure=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -n "${forced_package}" ]; then
  if [ -n "${count}" ] || \
    [ "${timeout}" -ne 0 ] || \
    [ -n "${prefered_package}" ] || \
    ${exit_after_failure}; then
    >&2 echo 'Conflicting flags.'
    usage
  fi
  upload_to_build_master=false
else
  upload_to_build_master=true
fi

if [ -z "${count}" ]; then
  if [ "${timeout}" -ne 0 ] || ${exit_after_failure}; then
    count=-1
  else
    count=1
  fi
fi

if [ -n "${forced_straws}" ]; then
  straws_that_might_repair_failing_builds="${forced_straws# }"
fi

if [ "${timeout}" -ne 0 ]; then
  timeout=$((timeout+$(date +%s)))
fi

checksum=$(
  calculate_script_checksum
)

# When this script or a script in lib/ got updated, we do not request
# any new assignments. This script should rather exit and be restarted.
while [ "${count}" -ne 0 ] && \
  [ "$(calculate_script_checksum)" = "${checksum}" ]; do

  if [ "${timeout}" -ne 0 ] && [ "${timeout}" -lt "$(date +%s)" ];
  then
    break
  fi

  err=0
  if [ -z "${forced_package}" ]; then
    package=$(
      # shellcheck disable=SC2029
      ssh \
        -i "${master_build_server_identity}" \
        -p "${master_build_server_port}" \
        "${master_build_server_user}@${master_build_server}" \
        'get-assignment' 'i686' "${prefered_package}"
    ) || err=$?
    if [ "$(printf '%s\n' "${package}" | wc -l)" -ne 1 ]; then
      expected_packages=$(
        printf '%s\n' "${package}" | \
          sed '1d'
      )
      package=$(
        printf '%s\n' "${package}" | \
          sed -n '1p'
      )
    else
      expected_packages=''
    fi
  else
    package=$(
      echo "${forced_package}" | \
        sed '
          s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
        '
    )
    expected_packages=''
  fi

  case ${err} in

    #  0: ok, I gave you an assignment
    0)
      [ ${count} -gt 0 ] && \
        count=$((count-1))
      sub_pkgrel="${package##* }"
      package="${package% *}"
      repository="${package##* }"
      package="${package% *}"
      mod_git_revision="${package##* }"
      package="${package% *}"
      git_revision="${package##* }"
      if [ "${git_revision}" = "${package}" ]; then
        # build master did not tell us a sub_pkgrel
        git_revision="${mod_git_revision}"
        mod_git_revision="${repository}"
        repository="${sub_pkgrel}"
        sub_pkgrel='0'
      else
        package="${package% *}"
      fi

      if [ "${git_revision##*-}" = 'HEAD' ]; then
        git_revision=$(
          repo_name="${git_revision%-*}"
          eval repo_path='"${repo_paths__'"${repo_name}"'}"'
          if [ -z "${repo_path}" ]; then
            >&2 printf 'Unknown git repository "%s".\n' "${repo_name}"
            exit 2
          fi
          git -C "${repo_path}" rev-parse HEAD
        )
      fi

      if [ "${mod_git_revision}" = 'work-tree' ]; then
        mod_git_revision=$(
          # we can't just create an empty index-file with mktemp, because git doesn't like it
          find . \
            -mindepth 1 \
            -maxdepth 1 \
            -name 'tmp.build-packages.git.*' \
            -exec rm -rf --one-file-system {} \;
          tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir)
          trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT
          export GIT_INDEX_FILE="${tmp_subdir}/index.new"
          git -C "${repo_paths__archlinux32}" add -A
          git -C "${repo_paths__archlinux32}" write-tree
        )
      fi

      # Update git repositories (official packages, community packages and the repository of package customizations).

      for repo_name in ${repo_names}; do
        eval repo_path='"${repo_paths__'"${repo_name}"'}"'
        git -C "${repo_path}" remote update
      done

      git_repo=$(find_repository_with_commit "${git_revision}")

      find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"

      # trigger update of mirror (if configured)
      if [ -n "${mirror_update_command}" ]; then
        ${mirror_update_command}
      fi

      bail_out() {
        err=$?
        if [ -n "$1" ]; then
          err="$1"
        fi
        cd "${base_dir}"
        recursively_umount_and_rm "${tmp_dir}"
        flock -u 9 || true
        exit "${err}"
      }
      find "${work_dir}" \
        -mindepth 1 \
        -maxdepth 1 \
        -name 'tmp.build-packages.??????' \
        -printf '%p\n' | \
        while read -r old_tmp_dir; do
          find "${old_tmp_dir}" -xdev -exec chmod 777 {} \;
          rm -rf --one-file-system "${old_tmp_dir}"
        done
      tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
      trap bail_out EXIT

      extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" "${sub_pkgrel}"

      cd "${tmp_dir}"
      echo 'nothing' > "${tmp_dir}/.ping-build-master"
      if [ -z "${forced_package}" ]; then
        # we get a lock on "${work_dir}/ping-build-master.lock",
        # if we release that lock, ping-to-master should stop _immediately_
        exec 9> "${work_dir}/ping-build-master.lock"
        if ! verbose_flock -n 9; then
          >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.'
          exit 2
        fi
        "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" &
      fi
      success=false
      for straw in ${straws_that_might_repair_failing_builds}; do

        echo 'preparing' > "${tmp_dir}/.ping-build-master"

        if echo "${straw}" | \
          grep -qF ':mirrored_source:'; then
          # maybe a missing source is/was the problem?
          if makepkg --verifysource 2>/dev/null; then
            # nope, sources are fine
            continue
          fi
          # try to download them from sources.archlinux.org/sources/$repo/$source
          source_name=$(
            makepkg --printsrcinfo | \
              sed -n '
                /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
                /^pkgname = /q
              ' | \
              sed '
                s|^pkgbase = \(.*\)$|0 \1-|
                s|^epoch = \(.*\)$|1 \1:|
                s|^pkgver = \(.*\)$|2 \1-|
                s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1.src.tar.gz|
              ' | \
              sort -k1n,1 | \
              sed '
                s|^[0-9] ||
                :a
                  N
                  s|\n[0-9] \(\S\+\)$|\1|
                ta
              '
          )
          if ! wget -q --timeout=15 -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
            # we can't improve anything
            continue
          fi
          # shellcheck disable=SC2046
          tar -xz --overwrite \
            -f "${source_name}" \
            --exclude PKGBUILD \
            $(
              if [ -n "${PKGBUILD_mod}" ]; then
                git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \
                  tar -t | \
                  sed 's/^/--exclude /'
              fi
            ) \
            --strip-components=1 \
            || true
        fi

        if echo "${straw}" | \
          grep -qF ':mirrored_source_by_hash:'; then
          # maybe a missing source is/was the problem?
          if makepkg --verifysource 2>/dev/null; then
            # nope, sources are fine
            continue
          fi
          # download it from sources.archlinux32.org by its hash
          if ! download_sources_by_hash "${package}" "${repository}" "${git_revision}" "${mod_git_revision}"; then
            # we can't improve anything, if no source was downloadable
            continue
          fi
        fi

        if echo "${straw}" | \
          grep -qF ':with_build_support:'; then
          build_command='staging-with-build-support-i686-build'
        else
          build_command='staging-i686-build'
        fi

        if echo "${straw}" | \
          grep -qF ':clean_chroot:'; then
          outerParameters='-c'
        else
          outerParameters=''
        fi

        if echo "${straw}" | \
          grep -qF ':without_check:'; then
          innerParameters='--nocheck'
        else
          innerParameters=''
        fi

        if echo "${straw}" | \
          grep -qF ':with_/dev/fuse:'; then
          middleParameters='-d /dev/fuse'
        else
          middleParameters=''
        fi

        find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \
          rm {} \;

        echo 'building' > "${tmp_dir}/.ping-build-master"
        >&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) ...' \
          "$(date +'%Y-%m-%d %T')" \
          "${package}" \
          "${git_revision}" \
          "${mod_git_revision}" \
          "${repository}" \
          "${straw}"
        # by piping the log, we don't see anything in the terminal,
        # but all ways to duplicate the logs seem pretty elaborate
        # shellcheck disable=SC2024,SC2086
        if sudo "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \
          "$(
            date -u --iso-8601=seconds | \
              cut -d+ -f1
          ).build-log" 2>&1; then
          # build successful
          >&2 printf ' ok.\n'
          tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX")
          find . -maxdepth 1 -type f -name '*-debug-*.pkg.tar.xz*' -delete
          echo 'post-build' > "${tmp_dir}/.ping-build-master"
          # remove unexpected packages
          if [ -n "${expected_packages}" ]; then
            {
              find . -maxdepth 1 -type f -name '*.pkg.tar.xz' -printf '%f\n'
              printf '%s\n' "${expected_packages}" | \
                sed 'p'
            } | \
              sort | \
              uniq -u | \
              while read -r unexpected_package; do
                rm "${unexpected_package}"*
              done
          fi
          >&2 printf 'signing package(s)\n'
          find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
            -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \
            -execdir mv '{}' '{}.sig' '{}-namcap.log' "${tar_content_dir}/" \; \
            -printf '%f\n' | \
            sponge | \
            while read -r pkg_file; do
              {
                pacman -Spdd --print-format '%l' --noconfirm "${pkg_file%-*-*-*}" | \
                  sed '
                    s|/[^/]\+\.pkg\.tar\.xz$||
                  '
                # shellcheck disable=SC2016
                curl -Ss 'https://www.archlinux.org/mirrorlist/?country=all&protocol=https&tier=1&use_mirror_status=on' | \
                  sed -n '
                    s/^#Server = //
                    T
                    s/\$repo/'"${repository}"'/g
                    s/\$arch/x86_64/g
                    p
                  ' | \
                  shuf
              } | \
                sed '
                  s|$|/'"${pkg_file}"'|
                  s/\.[0-9]\+\(-[^-]\+\)$/\1/
                  s/-i686\(\.pkg\.tar\.xz\)$/-x86_64\1/
                ' | \
                while read -r url; do
                  >&2 printf 'downloading "%s" ...' "${url}"
                  if wget -q --timeout=15 -nd "${url}"; then
                    >&2 printf ' ok.\n'
                    break;
                  fi
                  >&2 printf ' failed. Next ...\n'
                done
            done
          >&2 printf 'searching for provided libraries\n'
          find "${tar_content_dir}" -maxdepth 1 \
            -name '*.pkg.tar.xz' \
            -printf '%p\n' | \
            while read -r pkgfile; do
              pacman -Qqlp "${pkgfile}" | \
                sed -n '
                  s,^.*/,,
                  /\.so\(\..\+\)\?$/p
                ' > \
                "${pkgfile}.so.provides"
            done
          >&2 printf 'searching for required and more provided libraries\n'
          package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX")
          find "${tar_content_dir}" -maxdepth 1 \
            -name '*.pkg.tar.xz' | \
            while read -r pkgfile; do
              touch "${pkgfile}.so.needs"
              if printf '%s\n' "${pkgfile}" | \
                grep -vq -- '-any\.pkg\.tar\.xz$'; then
                # we do not check "any" packages for linked libraries
                # (why do they have them in the first place?)
                mkdir "${package_content_dir}/${pkgfile##*/}"
                tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
                find "${package_content_dir}/${pkgfile##*/}" \
                  -name 'opt' -prune , \
                  -exec objdump -x '{}' \; 2>/dev/null | \
                  sed '
                    /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
                  ' | \
                  grep -w 'NEEDED' | \
                  awk '{print $2}' | \
                  sed '
                    /\.c32$/d
                    s,^.*/,,
                  ' | \
                  sort -u > \
                  "${pkgfile}.so.needs"
                find "${package_content_dir}/${pkgfile##*/}" \
                  -name 'opt' -prune , \
                  -exec objdump -x '{}' \; 2>/dev/null | \
                  sed '
                    /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
                  ' | \
                  grep -w 'SONAME' | \
                  awk '{print $2}' | \
                  sort -u >> \
                  "${pkgfile}.so.provides"
                find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 '{}' \;
                rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
              fi
            done
          >&2 printf 'running namcap ...'
          if [ "${repository}" = 'multilib' ]; then
            x86_64_build_command='multilib-build'
          else
            x86_64_build_command='extra-x86_64-build'
          fi
          # this is a little hack: makepkg receives '--version', but namcap is run nevertheless
          # (and it only works with devtools32, because they are running namcap on *.pkg.tar.xz in the base directory, too)
          sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \
            sudo "${x86_64_build_command}" -c -- -- --version > /dev/null 2>&1 || \
            true
          >&2 printf ' ok.\n'
          >&2 printf 'smoothen namcap log ...'
          # now we generate diffs from the namcap.logs
          find . "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%p\n' | \
            while read -r log; do
              smoothen_namcap_log "${log}"
            done
          find "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%f\n' | \
            sed '
              s|\(^.*\)-i686\(\.pkg\.tar\.xz-namcap\.log\)$|\0 \1-x86_64\2|
              s|^.*-any\.pkg\.tar\.xz-namcap\.log$|\0 \0|
            ' | \
            while read -r log x86_64_log; do
              if [ -f "${x86_64_log}" ]; then
                diff -u "${x86_64_log}" "${tar_content_dir}/${log}" | \
                  sed '
                    1,3d
                    /^[^+-]/d
                  ' | \
                  sponge "${tar_content_dir}/${log}"
              else
                sed -i 's|^|*|' "${tar_content_dir}/${log}"
              fi
            done
          >&2 printf ' ok.\n'
          echo 'uploading' > "${tmp_dir}/.ping-build-master"
          if ${upload_to_build_master}; then
            find "${tar_content_dir}/" -maxdepth 1 \
              \( \
                -name '*.pkg.tar.xz-namcap.log' -o \
                -name '*.pkg.tar.xz.so.needs' -o \
                -name '*.pkg.tar.xz.so.provides' \
              \) \
              -execdir gzip '{}' \;
          else
            find "${tar_content_dir}/" -maxdepth 1 \
              -name '*.pkg.tar.xz-namcap.log' \
              -execdir grep -HF '' '{}' \;
          fi
          # shellcheck disable=SC2046
          tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
            find "${tar_content_dir}/" -maxdepth 1 \
              \( \
                -name '*.pkg.tar.xz' -o \
                -name '*.pkg.tar.xz.sig' -o \
                -name '*.pkg.tar.xz-namcap.log.gz' -o \
                -name '*.pkg.tar.xz.so.needs.gz' -o \
                -name '*.pkg.tar.xz.so.provides.gz' \
              \) \
              -printf '%f\n'
            )
          if ${upload_to_build_master}; then
            # shellcheck disable=SC2046
            rsync $(
              find "${tar_content_dir}/" -maxdepth 1 \
                \( \
                  -name '*.pkg.tar.xz' -o \
                  -name '*.pkg.tar.xz.sig' \
                \) \
              -printf '%p\n'
            ) "rsync://mirror.archlinux32.org/transfer32/" || true
          fi
          while ${upload_to_build_master}; do
            err=0
            # shellcheck disable=SC2029
            ssh \
              -i "${master_build_server_identity}" \
              -p "${master_build_server_port}" \
              "${master_build_server_user}@${master_build_server}" \
              'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${sub_pkgrel}" \
              < 'package.tar' || \
              err=$?
            case ${err} in
              0)
                # upload successful
                break
              ;;
              1)
                >&2 echo '"return-assignment" was running already.'
                wait_some_time 15
              ;;
              2)
                >&2 echo 'I was too slow, the package is outdated. I will continue ...'
                break
              ;;
              3)
                >&2 echo "'return-assignment' reports a signature error."
                exit 1
              ;;
              4)
                >&2 echo "'return-assignment' reports too many or missing packages."
                exit 1
              ;;
              *)
                >&2 echo "unknown return code ${err} from 'return-assignment'"
                wait_some_time 60
            esac
          done
          success=true
          break
        fi
        echo 'failure' > "${tmp_dir}/.ping-build-master"
        >&2 printf ' failed.\n'
      done

      if ! ${success}; then
        for log in *'.build-log'; do
          if [ -f "${log}" ]; then
            if ${upload_to_build_master}; then
              gzip "${log}"
            else
              grep -HF '' "${log}"
            fi
          fi
        done
        if ${upload_to_build_master} && \
          tar -cf 'build-logs.gz.tar' \
          -- *'.build-log.gz'; then
          while true; do
            err=0
            # shellcheck disable=SC2029
            ssh \
              -i "${master_build_server_identity}" \
              -p "${master_build_server_port}" \
              "${master_build_server_user}@${master_build_server}" \
              'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" 'ERROR' \
              < 'build-logs.gz.tar' || \
              err=$?
            case ${err} in
              0)
                # upload successful
                break
              ;;
              1)
                >&2 echo '"return-assignment" was running already.'
                wait_some_time 15
              ;;
              2)
                >&2 echo 'I was too slow, the package is outdated. I will continue ...'
                break
              ;;
              *)
                >&2 echo "unknown return code ${err} from 'return-assignment'"
                wait_some_time 60
            esac
          done
        fi
        if ${exit_after_failure}; then
          >&2 echo 'Build failed, exiting now'
          exit 0
        fi
      fi

      # clean up tmp_dir
      cd "${base_dir}"
      recursively_umount_and_rm "${tmp_dir}"
      flock -u 9 || true
      trap - EXIT

      continue

    ;;

    1)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' come back (shortly) later - I was running already'
      wait_some_time 15
      continue

    ;;

    2)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 2: come back later - there are still packages to be built,'
      >&2 echo ' but currently none has all its dependencies ready'
      wait_some_time 60
      continue

    ;;

    3)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 3: come back after the next run of get-package-updates - currently'
      >&2 echo ' there are no pending packages'
      exit 0

    ;;

    *)

      >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
      exit ${err}

    ;;

  esac

done

>&2 echo 'Done.'