diff options
| -rw-r--r-- | .Attic/README.old_stage2 | 60 | ||||
| -rw-r--r-- | README | 11 | ||||
| -rw-r--r-- | TODOS | 3 | ||||
| -rwxr-xr-x | build_stage2_package.sh | 4 | ||||
| -rwxr-xr-x | build_stage3.sh | 174 | ||||
| -rwxr-xr-x | build_stage3_package.sh | 168 | ||||
| -rw-r--r-- | default.conf | 15 | ||||
| -rw-r--r-- | i486-stage3/template/DESCR | 20 | ||||
| -rwxr-xr-x | prepare_stage3.sh | 28 |
9 files changed, 422 insertions, 61 deletions
diff --git a/.Attic/README.old_stage2 b/.Attic/README.old_stage2 index 6e7fc69..fceabbe 100644 --- a/.Attic/README.old_stage2 +++ b/.Attic/README.old_stage2 @@ -1,3 +1,5 @@ +# maybe some hints to be integrated into stage3.. + # old stage 2 based on a broken stage 1, so carefull what to pick # into the next version.. @@ -39,64 +41,8 @@ sed -i '/configure.ac/ a \ sed -i "s@\\(unit_SOURCES.*\\)@#\\1@g" Makefile.am' sed -i '/configure.ac/ a \ sed -i "s@\\(check_PROGRAMS*\\)@#\\1@g" Makefile.am' dejagnu/PKGBUILD sed -i '/Makefile.am/ a \ autoreconf' dejagnu/PKGBUILD -# glibc -####### -# use 2.25 version, not 2.26. Keep the ABI intact! -# endless loop? gawk -f ../scripts/gen-as-const.awk pthread-pi-defines.sym \ -# two reasons: clock and artifacts walk backwards or forward in time -# chroot on server with chroot, hard to have an asynchronous clock? -# or glibc is already installed on the system (where not?), so how the -# hell can it be possibly built! -# => oh, this would be cruel. -# or 3rd reason, we have a glibc shim, maybe this one causes some trouble -# => why? -# or libtool fixing, whatever fixing means (https://forums.gentoo.org/viewtopic-t-496052-start-0.html) -# => nope. -# -isystem /usr/includ ein gcc spec? quite likely! -# ah: old friend: -build/glibc/src/glibc-build/libc.a(dl-sysdep.o): In function `.L18': -dl-sysdep.c:(.text+0x8c): undefined reference to `__memcmp_ia32' -/build/glibc/src/glibc-build/libc.a(dl-sysdep.o): In function `.L87': -dl-sysdep.c:(.text+0x11f): undefined reference to `__memcmp_ia32' -/build/glibc/src/glibc-build/libc.a(dl-sysdep.o): In function `.L19': -dl-sysdep.c:(.text+0x152): undefined reference to `__memcmp_ia32' -/build/glibc/src/glibc-build/libc.a(dl-sysdep.o): In function `.L85': -dl-sysdep.c:(.text+0x19c): undefined reference to `__memcmp_ia32' -dl-sysdep.c:(.text+0x1b7): undefined reference to `__memcmp_ia32' -/build/glibc/src/glibc-build/libc.a(dl-sysdep.o):dl-sysdep.c:(.text+0x202): more undefined references to `__memcmp_ia32' follow -elf/sln -=> USE_MULTIARCH -=> --disable-multi-arch -=> let's test this time, too dangerous to loose the chroot otherwise! -# older 2.25 misses a binutils 2.29 patch: -https://git.busybox.net/buildroot/diff/package/glibc/0005-fix-binutils-2-29-build.patch?id=cf821efbd0b24690b52f379d4a9934a16073762e -{standard input}: Assembler messages: -{standard input}: Error: `loc1@GLIBC_2.0' can't be versioned to common symbol 'loc1' -{standard input}: Error: `loc2@GLIBC_2.0' can't be versioned to common symbol 'loc2' -{standard input}: Error: `locs@GLIBC_2.0' can't be versioned to common symbol 'locs' -make[2]: *** [../o-iterator.mk:9: /build/glibc/src/glibc-build/misc/regexp.os] Error 1 - PACKAGES=" \ -libgpg-error \ - \ -sysfsutils libidn iputils \ - tcl expect dejagnu gcc glibc" - -# aftermatch for gcc/binutils - -#collect2: unable to find ld: this sounds bad, like an archicture mismatch somewhere -#in binutils/gcc -=> this is the cross compiled gcc for i486 inside the chroot, it expects - its platform dependend stuff in /usr/lib/gcc/i486-unknown-linux-gnu/7.2.0 -=> we can temporarily fix this: -ln -s /usr/bin/ld /usr/lib/gcc/i486-unknown-linux-gnu/7.2.0/ld -etc. - -# aftermatch for glibc - -# bacause pacman cannot set file permissions -chmod u+x /usr/bin/* /lib/*.so* - + tcl expect dejagnu" # in 'real' 486 ############### @@ -191,3 +191,14 @@ su cross ./prepare_stage2_repo.sh # (we gave it 512 MB) ./build_stage2.sh + +######### +# STAGE 3 +######### + +# Use stage 2 to build base and base-devel. In this phase we don't care +# yet much about testing the packages (as this draws in still too many +# dependencies). But we try to be as close to base + base-devel as possible +# with minimal patching of PKGBUILDs. + +su cross ./prepare_stage3_repo.sh @@ -80,4 +80,7 @@ stage2 issues: make it a build requirement! - syslinux: linking issues, header section too small in ld linker script - syslinux: shouldn't we switch to another easier boot manager? or to grub? +- we lack a hdd and iso creator, for now we just take stage1 for stage2 and stage2 + for stage3 +stage 3 issues: diff --git a/build_stage2_package.sh b/build_stage2_package.sh index 95ab857..34c3826 100755 --- a/build_stage2_package.sh +++ b/build_stage2_package.sh @@ -3,7 +3,7 @@ # shellcheck source=./default.conf . "./default.conf" -# builds and installs one package for stage 1 +# builds and installs one package for stage 2 if test "$(id -u)" = 0; then sudo -u cross "$0" "$1" @@ -87,7 +87,7 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l -exec cp {} . \; fi - # execute makepkg on the host, we don't have git on the stage 1 machine (yet) + # execute makepkg on the host # we would actually like to have a mode like 'download, and noextract' but # makepkg is not doing that (see -e and -o options) diff --git a/build_stage3.sh b/build_stage3.sh new file mode 100755 index 0000000..18c9dcb --- /dev/null +++ b/build_stage3.sh @@ -0,0 +1,174 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# build all packages for stage 3 using the target system with stage 2 +# packages. packages will be installed with pacman onto the target +# system once built sucessfully. The artifacts are also copied back +# to the $STAGE3_PACKAGES to speed up rebuild of the state of the stage 2 +# system in case of destroying it. + +PACKAGES="which" + +# stage3 (from compute_dependencies.sh) +#~ acl: attr +#~ argon2: glibc +#~ attr: glibc +#~ autoconf: awk m4 diffutils sh +#~ automake: perl bash +#~ bash: readline glibc ncurses +#~ bc: readline +#~ binutils: glibc zlib +#~ bison: glibc m4 sh +#~ bzip2: glibc sh +#~ ca-certificates-cacert: ca-certificates-utils +#~ coreutils: glibc acl attr gmp libcap openssl +#~ cracklib: glibc zlib +#~ cryptsetup: device-mapper libgcrypt popt libutil-linux json-c argon2 +#~ curl: ca-certificates krb5 libssh2 openssl zlib libpsl libnghttp2 +#~ db: gcc-libs sh +#~ dbus: libsystemd expat +#~ dhcpcd: glibc sh udev libsystemd +#~ diffutils: glibc bash +#~ e2fsprogs: sh libutil-linux +#~ ed: glibc sh +#~ elfutils: gcc-libs zlib bzip2 xz +#~ expat: glibc +#~ fakeroot: glibc filesystem sed util-linux sh +#~ file: glibc zlib +#~ filesystem: iana-etc +#~ findutils: glibc sh +#~ flex: glibc m4 sh +#~ gawk: sh glibc mpfr +#~ gc: gcc-libs libatomic_ops +#~ gdbm: glibc sh +#~ gettext: gcc-libs acl sh glib2 libunistring +#~ glib2: pcre libffi libutil-linux zlib +#~ gmp: gcc-libs sh +#~ gnupg: npth libgpg-error libgcrypt libksba libassuan pinentry bzip2 readline gnutls sqlite +#~ gnutls: gcc-libs libtasn1 readline zlib nettle p11-kit libidn libunistring +#~ grep: glibc pcre +#~ groff: perl gcc-libs +#~ guile: gmp libltdl ncurses texinfo libunistring gc libffi +#~ guile2.0: gmp libltdl ncurses texinfo libunistring gc libffi +#~ gzip: glibc bash less +#~ inetutils: pam libcap +#~ iproute2: glibc iptables libelf +#~ iptables: glibc bash libnftnl libpcap +#~ iputils: openssl sysfsutils libcap libidn +#~ jfsutils: util-linux +#~ joe: ncurses +#~ json-c: glibc +#~ kbd: glibc pam +#~ keyutils: glibc sh +#~ kmod: glibc zlib xz +#~ krb5: e2fsprogs libldap keyutils +#~ ldns: openssl dnssec-anchors +#~ less: glibc ncurses pcre +#~ libarchive: acl attr bzip2 expat lz4 openssl xz zlib +#~ libassuan: libgpg-error +#~ libatomic_ops: glibc +#~ libcap: glibc attr +#~ libcap-ng: glibc +#~ libedit: ncurses +#~ libffi: glibc +#~ libgcrypt: libgpg-error +#~ libgpg-error: glibc sh +#~ libidn: glibc +#~ libidn2: libunistring +#~ libksba: bash libgpg-error glibc +#~ libmnl: glibc +#~ libmpc: mpfr +#~ libnfnetlink: glibc +#~ libnftnl: libmnl +#~ libnghttp2: glibc +#~ libnl: glibc +#~ libpcap: glibc libnl sh libusbx dbus +#~ libpipeline: glibc +#~ libpsl: libidn2 libunistring +#~ libseccomp: glibc +#~ libsecret: glib2 libgcrypt +#~ libssh2: openssl +#~ libtasn1: glibc +#~ libtirpc: krb5 +#~ libtool: sh tar glibc +#~ libunistring: glibc +#~ libunwind: glibc xz +#~ libusb: glibc libsystemd +#~ logrotate: popt gzip acl +#~ lz4: glibc +#~ m4: glibc bash +#~ make: glibc guile +#~ man-db: bash gdbm zlib groff libpipeline less libseccomp +#~ mdadm: glibc +#~ mkinitcpio: awk mkinitcpio-busybox kmod util-linux libarchive coreutils bash findutils grep filesystem gzip systemd +#~ mpfr: gmp +#~ nano: ncurses file sh +#~ nasm: glibc +#~ ncurses: glibc gcc-libs +#~ net-tools: glibc +#~ netctl: coreutils iproute2 openresolv systemd +#~ nettle: gmp +#~ openresolv: bash +#~ openssh: krb5 openssl libedit ldns +#~ openssl: perl +#~ p11-kit: glibc libtasn1 libffi +#~ pacman: bash glibc libarchive curl gpgme pacman-mirrorlist archlinux-keyring +#~ pam: glibc cracklib libtirpc pambase +#~ patch: glibc attr +#~ pciutils: glibc hwids kmod +#~ pcmciautils: systemd +#~ pcre: gcc-libs readline zlib bzip2 bash +#~ pcre2: gcc-libs readline zlib bzip2 bash +#~ perl: gdbm db glibc +#~ pinentry: ncurses libcap libassuan libsecret +#~ pkg-config: glib2 +#~ popt: glibc +#~ procps-ng: ncurses libsystemd +#~ psmisc: ncurses +#~ python: expat bzip2 gdbm openssl libffi zlib +#~ readline: glibc ncurses libncursesw.so +#~ reiserfsprogs: util-linux +#~ s-nail: openssl krb5 libidn +#~ sed: glibc acl attr +#~ shadow: bash pam acl +#~ strace: perl libunwind +#~ sudo: glibc libgcrypt pam libldap +#~ sysfsutils: glibc +#~ tar: glibc acl attr +#~ texinfo: ncurses gzip perl sh +#~ thin-provisioning-tools: expat gcc-libs libaio +#~ usbutils: libusb hwids +#~ vi: ncurses +#~ which: glibc bash +#~ xfsprogs: sh libutil-linux readline +#~ xz: sh +#~ zlib: glibc + +#~ stage2: +#~ bash +#~ iana-etc filesystem linux-api-headers tzdata +#~ ncurses readline joe +#~ attr acl m4 gmp gdbm db perl openssl +#~ libunistring gettext perl-locale-gettext help2man +#~ autoconf automake perl-error pcre2 git libtool +#~ zlib pambase cracklib libtirpc flex pam libcap coreutils +#~ util-linux pkg-config e2fsprogs expat bzip2 lz4 xz pcre less gzip +#~ tar libarchive curl +#~ pacman-mirrorlist archlinux-keyring archlinux32-keyring pacman +#~ elfutils sed texinfo grep findutils file diffutils ed patch +#~ fakeroot +#~ kbd procps-ng bison shadow +#~ inetutils bc kmod linux uinit nasm +#~ net-tools libmnl libnfnetlink iptables iproute2 +#~ libedit openssh +#~ make mpfr gawk libmpc binutils gcc glibc +#~ libunwind strace gdb +#~ " +#~ #TODO after nasm: syslinux + +for p in $PACKAGES; do + "$SCRIPT_DIR/build_stage3_package.sh" "$p" || exit 1 +done + diff --git a/build_stage3_package.sh b/build_stage3_package.sh new file mode 100755 index 0000000..fa2d018 --- /dev/null +++ b/build_stage3_package.sh @@ -0,0 +1,168 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# builds and installs one package for stage 3 + +if test "$(id -u)" = 0; then + sudo -u cross "$0" "$1" + exit 0 +fi + +PACKAGE=$1 + +# draw in default values for build variables + +. "$SCRIPT_DIR/$TARGET_CPU-stage3/template/DESCR" + +if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l)" = 0; then + echo "Building package $PACKAGE." + + cd $STAGE3_BUILD || exit 1 + + # clean up old build + + sudo rm -rf "$PACKAGE" + rm -f "$STAGE3_PACKAGES/$PACKAGE"-*pkg.tar.xz + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP rm -rf "/build/$PACKAGE" + + # check out the package build information from the upstream git rep + # using asp (or from the AUR using yaourt) + + PACKAGE_DIR="$SCRIPT_DIR/$TARGET_CPU-stage3/$PACKAGE" + PACKAGE_CONF="$PACKAGE_DIR/DESCR" + if test -f "$PACKAGE_CONF"; then + if test "$(grep -c FETCH_METHOD "$PACKAGE_CONF")" = 1; then + FETCH_METHOD=$(grep FETCH_METHOD "$PACKAGE_CONF" | cut -f 2 -d = | tr -d '"') + fi + fi + case $FETCH_METHOD in + "asp") + $ASP export "$PACKAGE" + ;; + "yaourt") + yaourt -G "$PACKAGE" + ;; + "packages32") + # (we assume, we only take core packages) + cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" . + ;; + *) + print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 + exit 1 + esac + + cd "$PACKAGE" || exit 1 + + # attach our destination platform to be a supported architecture + sed -i "/^arch=[^#]*any/!{/^arch=(/s/(/($TARGET_CPU /}" PKGBUILD + + # if there is a packages32 diff-PKGBUILD, attach it at the end + # (we assume, we build only 'core' packages during stage1) + DIFF_PKGBUILD="$ARCHLINUX32_PACKAGES/core/$PACKAGE/PKGBUILD" + if test -f "$DIFF_PKGBUILD"; then + cat "$DIFF_PKGBUILD" >> PKGBUILD + fi + + # copy all other files from Archlinux32, if they exist + # (we assume, we only take core packages during stage1) + if test -f "$DIFF_PKGBUILD"; then + find "$ARCHLINUX32_PACKAGES/core/$PACKAGE"/* ! -name PKGBUILD \ + -exec cp {} . \; + fi + + # source package descriptions, sets variables for this script + # and executes whatever is needed to build the package + + if test -f "$PACKAGE_CONF"; then + . "$PACKAGE_CONF" + fi + + # copy all files into the build area on the target machine + # (but the package DESCR file) + + if test -d "$PACKAGE_DIR"; then + find "$PACKAGE_DIR"/* ! -name DESCR \ + -exec cp {} . \; + fi + + # execute makepkg on the host + # we would actually like to have a mode like 'download, and noextract' but + # makepkg is not doing that (see -e and -o options) + + makepkg --nobuild + rm -rf "$STAGE3_BUILD/$PACKAGE/src" + + # copy everything to the stage 1 machine + + scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE3_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/. + + # building the actual package + + ssh -i $CROSS_HOME/.ssh/id_rsa build@$STAGE1_MACHINE_IP bash -c "'cd $PACKAGE && makepkg --skipchecksums --skippgpcheck --nocheck'" > $PACKAGE.log 2>&1 + RES=$? + + tail "$PACKAGE.log" + + if test $RES = 0; then + + # copy to our package folder in the first stage chroot + + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' + cd /build/$PACKAGE + rm -f ./*debug*.pkg.tar.xz + cp -v ./*.pkg.tar.xz /packages/$TARGET_CPU/. + '" + + # redo the whole pacman cache and repo (always running into trouble + # there, packages seem to reappear in old versions) + + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' + rm -rf /var/cache/pacman/pkg/* + rm -rf /packages/$TARGET_CPU/temp.db* + rm -rf /packages/$TARGET_CPU/temp.files* + repo-add /packages/$TARGET_CPU/temp.db.tar.gz /packages/$TARGET_CPU/*pkg.tar.xz + '" + + # install onto stage 1 system via pacman + + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' + # TODO: broken [temp] repo + #pacman --noconfirm -Syy $PACKAGE + pacman --noconfirm -U /packages/$TARGET_CPU/$PACKAGE-*.pkg.tar.xz + if test $ADDITIONAL_INSTALL_PACKAGE != ""; then + #pacman --noconfirm -Syy $ADDITIONAL_INSTALL_PACKAGE + pacman --noconfirm -U /packages/$TARGET_CPU/$ADDITIONAL_INSTALL_PACKAGE-*.pkg.tar.xz + fi + '" + + # copy packages from target machine and replace our local version with it + + tmp_dir=$(mktemp -d 'tmp.compute-dependencies.0.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + cd $STAGE3_BUILD || exit 1 + mv "$STAGE3_BUILD/$PACKAGE/$PACKAGE.log" "$tmp_dir" + cd "$STAGE3_BUILD" || exit 1 + rm -rf "$PACKAGE" + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'cd /build && tar zcf $PACKAGE.tar.gz $PACKAGE/'" + scp -i $CROSS_HOME/.ssh/id_rsa -rC build@$STAGE1_MACHINE_IP:/build/"$PACKAGE.tar.gz" "$STAGE3_BUILD/." + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'cd /build && rm -f $PACKAGE.tar.gz'" + tar zxf "$PACKAGE.tar.gz" + rm -f "$PACKAGE.tar.gz" + mv "$tmp_dir/$PACKAGE.log" "$STAGE3_BUILD/$PACKAGE/." + mv -vf "$STAGE3_BUILD/$PACKAGE/"*.pkg.tar.xz "$STAGE3_PACKAGES/." + + echo "Built package $PACKAGE." + + else + echo "ERROR building package $PACKAGE" + exit 1 + fi + + cd $STAGE3_BUILD || exit 1 + +else + echo "$PACKAGE exists." +fi diff --git a/default.conf b/default.conf index a8b82dc..b5a89ed 100644 --- a/default.conf +++ b/default.conf @@ -43,15 +43,26 @@ STAGE1_ISOLINUX=$CROSS_HOME/isolinux # where is stage1 installed on the target architecture and reachable STAGE1_MACHINE_IP=192.168.1.127 -# where packages are stored for stage 2 +# where build results are stored for stage 2 STAGE2_BUILD=$CROSS_HOME/$TARGET_CPU-build-stage2 -# the chroot of stage 2 +# the chroot of stage 2 (we don't use it acutally, we use a real +# machine to build the packages on), but it serves as storage +# location of final artifacts) STAGE2_CHROOT=$CROSS_HOME/$TARGET_CPU-root-stage2 # where packages are stored for stage 2 (on the host) STAGE2_PACKAGES=$STAGE2_CHROOT/packages/$TARGET_CPU/ +# the chroot of stage 3 +STAGE3_CHROOT=$CROSS_HOME/$TARGET_CPU-root-stage3 + +# where packages are stored for stage 3 (on the host) +STAGE3_PACKAGES=$STAGE3_CHROOT/packages/$TARGET_CPU/ + +# where build results are stored for stage 3 +STAGE3_BUILD=$CROSS_HOME/$TARGET_CPU-build-stage3 + # git repository for PKGBUILD diffs and patches for the # i686 architecture GIT_URL_ARCHLINUX32_PACKAGES=https://github.com/archlinux32/packages.git diff --git a/i486-stage3/template/DESCR b/i486-stage3/template/DESCR new file mode 100644 index 0000000..cb3d05e --- /dev/null +++ b/i486-stage3/template/DESCR @@ -0,0 +1,20 @@ +# FETCH_METHOD = "asp" | "yaourt" | "packages32" +# "asp" is the default tool to fetch the package description PKGBUILD and +# associated patch and other files. +# If the package exists only on AUR, then use "yaourt". +# If you want the package from Archlinux32, use "packages32". +# (this is only for where to get the base set of files from, the patches +# from packages32 are always applied) +FETCH_METHOD="asp" + +# NOPARALLEL_BUILD = 0 | 1 +# the -j<N> parameter to makepkg will be set to -j if +# NOPARALLEL_PACKAGE=1. The default is to use all avaiable cores +# and set -j<CPUS> +NOPARALLEL_BUILD=0 + +# ADDITIONAL_INSTALL_PACKAGE = <package name> +# per default the package has the same name as the package file, +# some packages generate additional package files to install +# (for example util-linux also has a libutil-linux) +ADDITIONAL_INSTALL_PACKAGE= diff --git a/prepare_stage3.sh b/prepare_stage3.sh new file mode 100755 index 0000000..7b4fdcb --- /dev/null +++ b/prepare_stage3.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# prepare the i486-build for stage 3 + +if test ! -d $STAGE3_CHROOT; then + mkdir $STAGE3_CHROOT + mkdir -p $STAGE3_PACKAGES +fi + +if test ! -d $STAGE3_BUILD; then + + # prepare the build enviroment + + mkdir $STAGE3_BUILD + cd $STAGE3_BUILD || exit 1 + + # TODO: actually build a stage2 hdd from the build artifacts of + # stage 2, for now we just copy the vm from stage1 after building + # and installing all packages from stage 2 and use it as new build + # machine. + + echo "Prepared the stage 3 build environment." +fi + +echo "Stage 3 ready." |