From 44f5b7af6444173b16d1495023d80aaef8a9c47a Mon Sep 17 00:00:00 2001 From: Andreas Baumann Date: Sun, 18 Feb 2018 21:40:12 +0100 Subject: work on stage 2 --- README | 23 +++++++++++ TODOS | 29 ++++++++----- build_stage2.sh | 17 ++++++++ build_stage2_package.sh | 101 +++++++++++++++++++++++++++++++++++++++++++++ create_hdd.sh | 20 ++++++++- default.conf | 18 +++++++- i486-stage2/bash/DESCR | 0 i486-stage2/template/DESCR | 20 +++++++++ prepare_stage2_repo.sh | 27 ++++++++++++ 9 files changed, 242 insertions(+), 13 deletions(-) create mode 100755 build_stage2.sh create mode 100755 build_stage2_package.sh create mode 100644 i486-stage2/bash/DESCR create mode 100644 i486-stage2/template/DESCR create mode 100755 prepare_stage2_repo.sh diff --git a/README b/README index f17ece6..9af3759 100644 --- a/README +++ b/README @@ -163,3 +163,26 @@ su cross ./create_cdrom.sh # stage 2 build system on the target architecture) su cross ./create_hdd.sh + +######### +# STAGE 2 +######### + +# Assume stage1 of the system is installed on the target architecture +# (virtual or a real machine), we have a key-based SSH connection to +# the machine and can build packages there. + +# Build stage2 in $STAGE2_BUILD with the tools on the stage1 system +# and modified PKGBUILDs and patches into the target system (replacing +# stage 1 packages). Stage 1 serves now the same function as the sysroot. + +# The goal is to get a self-hosting system on the target architecture, +# not necesseraly containing all of base and base-devel, but enough to +# bootstrap itself on the target architecture. + +su cross ./prepare_stage2_repo.sh + +# Build stage 2 on the target architecture and install it onto the +# stage 1 system. Resulting artifacts get stored in $STAGE2_BUILD. + +./build_stage2.sh diff --git a/TODOS b/TODOS index 56aa715..e1cb7c8 100644 --- a/TODOS +++ b/TODOS @@ -1,3 +1,5 @@ +general bugs: + - how to clean up sysroot nicely without having to rebuild the cross-compiler? for now only removing and stowing a copy somewhere before building stage1 helps. @@ -12,8 +14,6 @@ bsdtar: /home/cross/i486-root/packages/i486/linux-headers-4.15.1-2-i486.pkg.tar.xz: Not found in archive bsdtar: Error exit delayed from previous errors. Built package linux. -- currently gcc has a PKGBUILD modified by hand instead of diff seds in DESCR - (the diff is very big and complex) - removing all pacman databases and recreating them with repo-add, also removing all installed pacman packages seem a little bit paranoic. Find out how to speed this up and do it properly. @@ -29,19 +29,12 @@ error: no usable package repositories configured. installing and listing packages by hand works without problems. debug: unregistering database 'local' -- make: recursive tarkets like all-recursive don't work, neither with cross-compiled - make nor with the recompiled make on the stage1 system. - => rebuilding bash on the target works, but it's a workaround: - https://unix.stackexchange.com/questions/389022/make-unable-to-recurse-for-autoconf-like-projects/423448#423448 - some packages get build more than once in stage1 (linux-api-headers, pacman-mirrorlist) - make a wrapper for sed in DESCR, which is actually able to detect whether the patch was successful or not. Avoid hard to detect errors, when the upstream PKGBUILD(s) change. -- gdb uses cross-compiler source files for debugging: - Temporary breakpoint 1, 0x004005b0 in main () - (gdb) list - 1 /home/cross/.build/i486-unknown-linux-gnu/src/gcc/libgcc/libgcc2.c: No such file or directory. -- stage1 + +stage1 issues: - stage1: cdrom installs keyrings without having a gpg binary - 32 MB is not enough when installing packages => add a swap as first action! - stage1 has no bootloader package (syslinux) to install @@ -49,3 +42,17 @@ - using the cross compiler for syslinux is most likely overkill - some packages still build more than really needed: - syslinux: builds docu with asciidoc and build EFI stuff from gnu-efi +- rename STAGE1 directories i486-root to i486-stage1-root and i486-build + to i486-build-stage1 +- gdb uses cross-compiler source files for debugging: + Temporary breakpoint 1, 0x004005b0 in main () + (gdb) list + 1 /home/cross/.build/i486-unknown-linux-gnu/src/gcc/libgcc/libgcc2.c: No such file or directory. +- currently gcc has a PKGBUILD modified by hand instead of diff seds in DESCR + (the diff is very big and complex) + +stage2 issues: +- make: recursive tarkets like all-recursive don't work, neither with cross-compiled + make nor with the recompiled make on the stage1 system. + => rebuilding bash on the target works, but it's a workaround: + https://unix.stackexchange.com/questions/389022/make-unable-to-recurse-for-autoconf-like-projects/423448#423448 diff --git a/build_stage2.sh b/build_stage2.sh new file mode 100755 index 0000000..8cf5dfd --- /dev/null +++ b/build_stage2.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# build all packages for stage 2 using the target system with stage 1 +# packages. packages will be installed with pacman onto the target +# system once built sucessfully. The artifacts are also copied back +# to the $STAGE2_PACKAGES to speed up rebuild of the state of the stage 1 +# system in case of destroying it. + +PACKAGES="bash" + +for p in $PACKAGES; do + "$SCRIPT_DIR/build_stage2_package.sh" "$p" || exit 1 +done + diff --git a/build_stage2_package.sh b/build_stage2_package.sh new file mode 100755 index 0000000..ebf75da --- /dev/null +++ b/build_stage2_package.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# builds and installs one package for stage 1 + +if test "$(id -u)" = 0; then + sudo -u cross "$0" "$1" + exit 0 +fi + +PACKAGE=$1 + +# draw in default values for build variables + +. "$SCRIPT_DIR/$TARGET_CPU-stage2/template/DESCR" + +if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz"n | wc -l)" = 0; then + echo "Building package $PACKAGE." + + cd $STAGE2_BUILD || exit 1 + + # clean up old build + + sudo rm -rf "$PACKAGE" + rm -f "$STAGE2_PACKAGES/$PACKAGE"-*pkg.tar.xz + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP rm -rf "/build/$PACKAGE" + + # check out the package build information from the upstream git rep + # using asp (or from the AUR using yaourt) + + PACKAGE_DIR="$SCRIPT_DIR/$TARGET_CPU-stage2/$PACKAGE" + PACKAGE_CONF="$PACKAGE_DIR/DESCR" + if test -f "$PACKAGE_CONF"; then + if test "$(grep -c FETCH_METHOD "$PACKAGE_CONF")" = 1; then + FETCH_METHOD=$(grep FETCH_METHOD "$PACKAGE_CONF" | cut -f 2 -d = | tr -d '"') + fi + fi + case $FETCH_METHOD in + "asp") + $ASP export "$PACKAGE" + ;; + "yaourt") + yaourt -G "$PACKAGE" + ;; + "packages32") + # (we assume, we only take core packages) + cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" . + ;; + *) + print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 + exit 1 + esac + + cd "$PACKAGE" || exit 1 + + # attach our destination platform to be a supported architecture + sed -i "/^arch=[^#]*any/!{/^arch=(/s/(/($TARGET_CPU /}" PKGBUILD + + # if there is a packages32 diff-PKGBUILD, attach it at the end + # (we assume, we build only 'core' packages during stage1) + DIFF_PKGBUILD="$ARCHLINUX32_PACKAGES/core/$PACKAGE/PKGBUILD" + if test -f "$DIFF_PKGBUILD"; then + cat "$DIFF_PKGBUILD" >> PKGBUILD + fi + + # copy all other files from Archlinux32, if they exist + # (we assume, we only take core packages during stage1) + if test -f "$DIFF_PKGBUILD"; then + find "$ARCHLINUX32_PACKAGES/core/$PACKAGE"/* ! -name PKGBUILD \ + -exec cp {} . \; + fi + + # source package descriptions, sets variables for this script + # and executes whatever is needed to build the package + + if test -f "$PACKAGE_CONF"; then + . "$PACKAGE_CONF" + fi + + # copy all files into the build area on the target machine + # (but the package DESCR file) + if test -d "$PACKAGE_DIR"; then + find "$PACKAGE_DIR"/* ! -name DESCR \ + -exec cp {} . \; + fi + + # execute makepkg on the host, we don't have git on the stage 1 machine (yet) + makepkg --nobuild + + # copy everything to the stage 1 machine + scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE2_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/. + + # TODO: + # building the actual package + #$STAGE1_BUILD/makepkg-$TARGET_CPU -C --config $STAGE1_BUILD/makepkg-$TARGET_CPU.conf \ +# --skipchecksums --skippgpcheck --nocheck > "$PACKAGE.log" 2>&1 +# RES=$? + +fi diff --git a/create_hdd.sh b/create_hdd.sh index 3170a64..94df476 100755 --- a/create_hdd.sh +++ b/create_hdd.sh @@ -53,7 +53,7 @@ mount -t devpts devpts /dev/pts mount -t sysfs sys /sys mount -o remount,rw / ip link set up dev eth0 -ip addr add 192.168.1.127/24 dev eth0 +ip addr add ${STAGE1_MACHINE_IP}/24 dev eth0 ip route add default via 192.168.1.1 dev eth0 /usr/sbin/sshd exec /usr/bin/bash @@ -81,6 +81,21 @@ chmod 0400 etc/ssh/ssh_host_*_key mkdir root/.ssh cp "$HOME/.ssh/id_rsa.pub" root/.ssh/authorized_keys +# install a build user and build directory +cat >> etc/group <> etc/passwd < root/test.c < parameter to makepkg will be set to -j if +# NOPARALLEL_PACKAGE=1. The default is to use all avaiable cores +# and set -j +NOPARALLEL_BUILD=0 + +# ADDITIONAL_INSTALL_PACKAGE = +# per default the package has the same name as the package file, +# some packages generate additional package files to install +# (for example util-linux also has a libutil-linux) +ADDITIONAL_INSTALL_PACKAGE= diff --git a/prepare_stage2_repo.sh b/prepare_stage2_repo.sh new file mode 100755 index 0000000..5ce5f00 --- /dev/null +++ b/prepare_stage2_repo.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# prepare the i486-build for stage 2 + +if test ! -d $STAGE2_CHROOT; then + mkdir $STAGE2_CHROOT + mkdir -p $STAGE2_PACKAGES +fi + +if test ! -d $STAGE2_BUILD; then + + # prepare the build enviroment + + mkdir $STAGE2_BUILD + cd $STAGE2_BUILD || exit 1 + + # TODO: check makepkg in stage1 hdd + # TODO: makepkg patch to run as root or add a build user to hdd + # TODO: we assume we have on cpu for now on the target + + echo "Prepared the stage 2 build environment." +fi + +echo "Stage 2 ready." -- cgit v1.2.3-54-g00ecf