diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 9 | ||||
| -rw-r--r-- | arch-nspawn.in | 2 | ||||
| -rw-r--r-- | archbuild.in | 6 | ||||
| -rw-r--r-- | doc/devtools.7.asciidoc | 46 | ||||
| -rw-r--r-- | doc/makerepropkg.1.asciidoc | 11 | ||||
| -rw-r--r-- | doc/offload-build.1.asciidoc | 2 | ||||
| -rw-r--r-- | makechrootpkg.in | 17 | ||||
| -rw-r--r-- | makepkg-x86_64.conf | 4 | ||||
| -rwxr-xr-x | makerepropkg.in | 63 | ||||
| -rwxr-xr-x | offload-build.in | 8 |
11 files changed, 137 insertions, 32 deletions
@@ -19,6 +19,7 @@ crossrepomove arch-nspawn sogrep doc/*.1 +doc/*.7 pacman-*-i486.conf pacman-*-i686.conf pacman-*-pentium4.conf @@ -120,7 +120,8 @@ MANS = \ doc/makerepropkg.1 \ doc/mkarchroot.1 \ doc/find-libdeps.1 \ - doc/find-libprovides.1 + doc/find-libprovides.1 \ + doc/devtools.7 all: $(GENERATED_CONFIGFILES) $(BINPROGS) bash_completion zsh_completion man @@ -130,11 +131,11 @@ edit = sed -e "s|@pkgdatadir[@]|$(PREFIX)/share/devtools|g" makepkg-i486.conf: makepkg-x86_64.conf @echo "GEN $@" - @sed 's,\(["=]\)x86[-_]64\([-" ]\),\1i486\2,g' "$<" > "$@" + @sed 's,\(["=]\)x86[-_]64\([-" ]\),\1i486\2,g; s,-fcf-protection,,g;' "$<" > "$@" makepkg-i686.conf: makepkg-x86_64.conf @echo "GEN $@" - @sed 's,\(["=]\)x86[-_]64\([-" ]\),\1i686\2,g' "$<" > "$@" + @sed 's,\(["=]\)x86[-_]64\([-" ]\),\1i686\2,g; s,-fcf-protection,,g; ' "$<" > "$@" makepkg-pentium4.conf: makepkg-i686.conf @echo "GEN $@" @@ -221,7 +222,7 @@ dist: gpg --detach-sign --use-agent devtools32-$(V).tar.gz upload: - scp devtools32-$(V).tar.gz devtools32-$(V).tar.gz.sig sources.archlinux32.org:sources/ + scp devtools32-$(V).tar.gz devtools32-$(V).tar.gz.sig sources.archlinux32.org:httpdocs/sources/ check: $(BINPROGS) bash_completion makepkg-x86_64.conf PKGBUILD.proto shellcheck $^ diff --git a/arch-nspawn.in b/arch-nspawn.in index b9215a6..07ccdc9 100644 --- a/arch-nspawn.in +++ b/arch-nspawn.in @@ -64,7 +64,7 @@ if (( ${#cache_dirs[@]} == 0 )); then fi # shellcheck disable=2016 -host_mirrors=($($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#;t;s#(.*/)[^/]+/extra$#\1$arch/$repo#')) +host_mirrors=($($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#;t;s#(.*/)[^/]+/extra(/?)$#\1$arch/$repo\2#')) for host_mirror in "${host_mirrors[@]}"; do if [[ $host_mirror == *file://* ]]; then diff --git a/archbuild.in b/archbuild.in index e7733c8..827ee66 100644 --- a/archbuild.in +++ b/archbuild.in @@ -76,6 +76,12 @@ if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then lock 9 "$copy.lock" "Locking chroot copy '%s'" "$copy" subvolume_delete_recursive "${copy}" + find "${copy}" \ + -mindepth 1 \ + -type d \ + -exec mountpoint -q {} \; \ + -prune \ + -exec umount -l {} \; rm -rf --one-file-system "${copy}" done lock_close 9 diff --git a/doc/devtools.7.asciidoc b/doc/devtools.7.asciidoc new file mode 100644 index 0000000..1b77f4e --- /dev/null +++ b/doc/devtools.7.asciidoc @@ -0,0 +1,46 @@ +devtools(7) +=========== + +Name +---- +devtools - Developer tools for the Arch Linux distribution + +Description +----------- + +Devtools contains tools for package maintenance in Arch Linux. The toolset +varies from tools for building packages in a clean chroot ('mkarchroot',...), +packaging related tools for sonames ('sogrep', 'lddd') and tools for +repository management such as ('archco', 'extra2community') + +Programs +-------- +The list below gives a short overview; see the respective documentation +for details. + +linkman:checkpkg[1] + Compare the current build pakcage with the repository version + +linkman:find-libdeps[1] + Find soname dependencies for a package + +linkman:find-libprovides[1] + Find soname's which are provided by a package + +linkman:lddd[1] + Find broken library links on your system + +linkman:makerepropkg[1] + Rebuild a package to see if it is reproducible + +linkman:mkarchroot[1] + Creates an arch chroot in a specified location with a specified set of + packages + +linkman:offload-build[1] + Build a PKGBUILD on a remote server using makechrootpkg + +linkman:sogrep[1] + Find packages using a linked to a given shared library + +include::footer.asciidoc[] diff --git a/doc/makerepropkg.1.asciidoc b/doc/makerepropkg.1.asciidoc index 0d7ddcb..e9f4c24 100644 --- a/doc/makerepropkg.1.asciidoc +++ b/doc/makerepropkg.1.asciidoc @@ -7,7 +7,7 @@ makerepropkg - Rebuild a package to see if it is reproducible Synopsis -------- -makerepropkg [OPTIONS] <package_file>... +makerepropkg [OPTIONS] [<package_file|pkgname>...] Description ----------- @@ -24,6 +24,15 @@ When given multiple packages, additional package files are assumed to be split packages and will be treated as additional artifacts to compare during the verification step. +A valid target(s) for pacman -S can be specified instead, and makerepropkg will +download it to the cache if needed. This is mostly useful to specify which +repository to retrieve from. If no positional arguments are specified, the +targets will be sourced from the PKGBUILD. + +In either case, the package name will be converted to a filename from the +cache, and makerepropkg will proceed as though this filename was initially +specified. + This implements a verifier for pacman/libalpm packages in accordance with the link:https://reproducible-builds.org/[Reproducible Builds] project. diff --git a/doc/offload-build.1.asciidoc b/doc/offload-build.1.asciidoc index 3633faa..5c9515f 100644 --- a/doc/offload-build.1.asciidoc +++ b/doc/offload-build.1.asciidoc @@ -28,7 +28,7 @@ Options architecture officially supported by Arch Linux. *-s, --server* <hostname>:: - Offload to a specific build server. The default is dragon.archlinux.org + Offload to a specific build server. The default is build.archlinux.org which is used as part of the build toolchain for the official Arch Linux repos. diff --git a/makechrootpkg.in b/makechrootpkg.in index affc5bc..456c25a 100644 --- a/makechrootpkg.in +++ b/makechrootpkg.in @@ -251,6 +251,15 @@ download_sources() { die "Could not download sources." } +move_logfiles() { + local l + for l in "$copydir"/logdest/*; do + [[ $l == */logpipe.* ]] && continue + chown "$src_owner" "$l" + mv "$l" "$LOGDEST" + done +} + move_products() { local pkgfile for pkgfile in "$copydir"/pkgdest/*; do @@ -263,12 +272,7 @@ move_products() { fi done - local l - for l in "$copydir"/logdest/*; do - [[ $l == */logpipe.* ]] && continue - chown "$src_owner" "$l" - mv "$l" "$LOGDEST" - done + move_logfiles for s in "$copydir"/srcpkgdest/*; do chown "$src_owner" "$s" @@ -380,6 +384,7 @@ then move_products else (( ret += 1 )) + move_logfiles fi (( temp_chroot )) && delete_chroot "$copydir" "$copy" diff --git a/makepkg-x86_64.conf b/makepkg-x86_64.conf index a07756c..6260a0d 100644 --- a/makepkg-x86_64.conf +++ b/makepkg-x86_64.conf @@ -15,7 +15,7 @@ DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u' 'ftp::/usr/bin/curl -gqfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' 'http::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' 'https::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'rsync::/usr/bin/rsync --no-motd -z %u %o' + 'rsync::/usr/bin/rsync --no-motd -zz %u %o' 'scp::/usr/bin/scp -C %u %o') # Other common tools: @@ -132,7 +132,7 @@ DBGSRCDIR="/usr/src/debug" COMPRESSGZ=(gzip -c -f -n) COMPRESSBZ2=(bzip2 -c -f) COMPRESSXZ=(xz -c -z -) -COMPRESSZST=(zstd -c -T0 --ultra -20 -) +COMPRESSZST=(zstd -c -z -q -) COMPRESSLRZ=(lrzip -q) COMPRESSLZO=(lzop -q) COMPRESSZ=(compress -c -f) diff --git a/makerepropkg.in b/makerepropkg.in index 62ba5f3..7d6ae41 100755 --- a/makerepropkg.in +++ b/makerepropkg.in @@ -57,9 +57,15 @@ parse_buildinfo() { get_pkgfile() { local cdir=${cache_dirs[0]} local pkgfilebase=${1} + local mode=${2} local pkgname=${pkgfilebase%-*-*-*} local pkgfile ext + # try without downloading + if [[ ${mode} != localonly ]] && get_pkgfile "${pkgfilebase}" localonly; then + return 0 + fi + for ext in .zst .xz ''; do pkgfile=${pkgfilebase}.pkg.tar${ext} @@ -72,6 +78,9 @@ get_pkgfile() { for f in "${pkgfile}" "${pkgfile}.sig"; do if [[ ! -f "${cdir}/${f}" ]]; then + if [[ ${mode} = localonly ]]; then + continue 2 + fi msg2 "retrieving '%s'..." "${f}" >&2 curl -Llf -# -o "${cdir}/${f}" "${archiveurl}/${pkgname:0:1}/${pkgname}/${f}" || continue 2 fi @@ -115,20 +124,46 @@ shift $((OPTIND - 1)) check_root -if [[ -n $1 ]]; then - pkgfile="$1" - splitpkgs=("$@") - for f in "${splitpkgs[@]}"; do - if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then - error "file is not a valid pacman package: '%s'" "${f}" - exit 1 - fi - done -else - error "no package file specified. Try '${BASH_SOURCE[0]##*/} -h' for more information. " - exit 1 +[[ -f PKGBUILD ]] || { error "No PKGBUILD in current directory."; exit 1; } + +# without arguments, get list of packages from PKGBUILD +if [[ -z $1 ]]; then + mapfile -t pkgnames < <(source PKGBUILD; pacman -Sddp --print-format '%r/%n' "${pkgname[@]}") + wait $! || { + error "No package file specified and failed to retrieve package names from './PKGBUILD'." + plain "Try '${BASH_SOURCE[0]##*/} -h' for more information." >&2 + exit 1 + } + msg "Reproducing all pkgnames listed in ./PKGBUILD" + set -- "${pkgnames[@]}" fi +# check each package to see if it's a file, and if not, try to download it +# using pacman -Sw, and get the filename from there +splitpkgs=() +for p in "$@"; do + if [[ -f ${p} ]]; then + splitpkgs+=("${p}") + else + pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) || { error "package name '%s' not in repos" "${p}"; exit 1; } + pkgfile=${pkgfile_remote#file://} + if [[ ! -f ${pkgfile} ]]; then + msg "Downloading package '%s' into pacman's cache" "${pkgfile}" + sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1 + pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) + pkgfile="${pkgfile_remote#file://}" + fi + splitpkgs+=("${pkgfile}") + fi +done + +for f in "${splitpkgs[@]}"; do + if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then + error "file is not a valid pacman package: '%s'" "${f}" + exit 1 + fi +done + if (( ${#cache_dirs[@]} == 0 )); then mapfile -t cache_dirs < <(pacman-conf CacheDir) fi @@ -139,11 +174,11 @@ load_makepkg_config HOME=${ORIG_HOME} [[ -d ${SRCDEST} ]] || SRCDEST=${PWD} -parse_buildinfo < <(bsdtar -xOqf "${pkgfile}" .BUILDINFO) +parse_buildinfo < <(bsdtar -xOqf "${splitpkgs[0]}" .BUILDINFO) export SOURCE_DATE_EPOCH="${buildinfo[builddate]}" PACKAGER="${buildinfo[packager]}" BUILDDIR="${buildinfo[builddir]}" -PKGEXT=${pkgfile#${pkgfile%.pkg.tar*}} +PKGEXT=${splitpkgs[0]#${splitpkgs[0]%.pkg.tar*}} # nuke and restore reproducible testenv for copy in "${buildroot}"/*/; do diff --git a/offload-build.in b/offload-build.in index 92f400c..d40b004 100755 --- a/offload-build.in +++ b/offload-build.in @@ -24,7 +24,7 @@ source /usr/share/makepkg/util/config.sh # global defaults suitable for use by Arch staff repo=extra arch=x86_64 -server=dragon.archlinux.org +server=build.archlinux.org die() { printf "error: $1\n" "${@:2}"; exit 1; } @@ -83,6 +83,9 @@ archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@") trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT +# Load makepkg.conf variables to be available +load_makepkg_config + # Use a source-only tarball as an intermediate to transfer files. This # guarantees the checksums are okay, and guarantees that all needed files are # transferred, including local sources, install scripts, and changelogs. @@ -98,7 +101,7 @@ mapfile -t files < <( # This is sort of bash golfing but it allows running a mildly complex # command over ssh with a single connection. # shellcheck disable=SC2145 - cat "$SRCPKGDEST"/*.src.tar.gz | + cat "$SRCPKGDEST"/*"$SRCEXT" | ssh $server ' temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" && mkdir -p "$temp" && @@ -122,7 +125,6 @@ mapfile -t files < <( if (( ${#files[@]} )); then printf '%s\n' '' '-> copying files...' - load_makepkg_config scp "${files[@]/#/$server:}" "${TEMPDIR}/" mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/" mv "${TEMPDIR}/PKGBUILD" "${PWD}/" |