From ba4812f637c3fc034370d7b3c6d7b1cd596ed72e Mon Sep 17 00:00:00 2001 From: Adolfo Delorenzo Date: Thu, 12 Feb 2026 17:20:04 -0600 Subject: [PATCH] fix: complete ARM64 RPi build pipeline - fetch-components.sh: download ARM64 KubeSolo binary (kubesolo-arm64) - inject-kubesolo.sh: use arch-specific binaries for KubeSolo, cloud-init, and update agent; detect KVER from custom kernel when rootfs has none; cross-arch module resolution via find fallback when modprobe fails - create-rpi-image.sh: kpartx support for Docker container builds - Makefile: rootfs-arm64 depends on build-cross, includes pack-initramfs Co-Authored-By: Claude Opus 4.6 --- Makefile | 4 +- build/scripts/create-rpi-image.sh | 50 +++++++++++++++++++----- build/scripts/fetch-components.sh | 33 ++++++++++++++++ build/scripts/inject-kubesolo.sh | 65 +++++++++++++++++++++++++------ 4 files changed, 130 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index ad908bd..ea64bf3 100644 --- a/Makefile +++ b/Makefile @@ -79,11 +79,13 @@ kernel-arm64: @echo "==> Building ARM64 kernel for Raspberry Pi..." $(BUILD_DIR)/scripts/build-kernel-arm64.sh -rootfs-arm64: +rootfs-arm64: build-cross @echo "==> Preparing ARM64 rootfs..." TARGET_ARCH=arm64 $(BUILD_DIR)/scripts/fetch-components.sh TARGET_ARCH=arm64 $(BUILD_DIR)/scripts/extract-core.sh TARGET_ARCH=arm64 $(BUILD_DIR)/scripts/inject-kubesolo.sh + @echo "==> Packing ARM64 initramfs..." + $(BUILD_DIR)/scripts/pack-initramfs.sh rpi-image: rootfs-arm64 kernel-arm64 @echo "==> Creating Raspberry Pi SD card image..." diff --git a/build/scripts/create-rpi-image.sh b/build/scripts/create-rpi-image.sh index b7872a2..1f17daa 100755 --- a/build/scripts/create-rpi-image.sh +++ b/build/scripts/create-rpi-image.sh @@ -83,9 +83,38 @@ type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, name="Data" EOF # --- Set up loop device --- -LOOP=$(losetup --show -fP "$IMG_OUTPUT") +LOOP=$(losetup --show -f "$IMG_OUTPUT") echo "==> Loop device: $LOOP" +# Use kpartx for reliable partition device nodes (works in Docker/containers) +USE_KPARTX=false +if [ ! -b "${LOOP}p1" ]; then + if command -v kpartx >/dev/null 2>&1; then + kpartx -a "$LOOP" + USE_KPARTX=true + sleep 1 + LOOP_NAME=$(basename "$LOOP") + P1="/dev/mapper/${LOOP_NAME}p1" + P2="/dev/mapper/${LOOP_NAME}p2" + P3="/dev/mapper/${LOOP_NAME}p3" + P4="/dev/mapper/${LOOP_NAME}p4" + else + # Retry with -P flag + losetup -d "$LOOP" + LOOP=$(losetup --show -fP "$IMG_OUTPUT") + sleep 1 + P1="${LOOP}p1" + P2="${LOOP}p2" + P3="${LOOP}p3" + P4="${LOOP}p4" + fi +else + P1="${LOOP}p1" + P2="${LOOP}p2" + P3="${LOOP}p3" + P4="${LOOP}p4" +fi + MNT_CTL=$(mktemp -d) MNT_BOOTA=$(mktemp -d) MNT_BOOTB=$(mktemp -d) @@ -96,22 +125,25 @@ cleanup() { umount "$MNT_BOOTA" 2>/dev/null || true umount "$MNT_BOOTB" 2>/dev/null || true umount "$MNT_DATA" 2>/dev/null || true + if [ "$USE_KPARTX" = true ]; then + kpartx -d "$LOOP" 2>/dev/null || true + fi losetup -d "$LOOP" 2>/dev/null || true rm -rf "$MNT_CTL" "$MNT_BOOTA" "$MNT_BOOTB" "$MNT_DATA" 2>/dev/null || true } trap cleanup EXIT # --- Format partitions --- -mkfs.vfat -F 32 -n KSOLOCTL "${LOOP}p1" -mkfs.vfat -F 32 -n KSOLOA "${LOOP}p2" -mkfs.vfat -F 32 -n KSOLOB "${LOOP}p3" -mkfs.ext4 -q -L KSOLODATA "${LOOP}p4" +mkfs.vfat -F 32 -n KSOLOCTL "$P1" +mkfs.vfat -F 32 -n KSOLOA "$P2" +mkfs.vfat -F 32 -n KSOLOB "$P3" +mkfs.ext4 -q -L KSOLODATA "$P4" # --- Mount all partitions --- -mount "${LOOP}p1" "$MNT_CTL" -mount "${LOOP}p2" "$MNT_BOOTA" -mount "${LOOP}p3" "$MNT_BOOTB" -mount "${LOOP}p4" "$MNT_DATA" +mount "$P1" "$MNT_CTL" +mount "$P2" "$MNT_BOOTA" +mount "$P3" "$MNT_BOOTB" +mount "$P4" "$MNT_DATA" # --- Boot Control Partition (KSOLOCTL) --- echo " Writing autoboot.txt..." diff --git a/build/scripts/fetch-components.sh b/build/scripts/fetch-components.sh index c01a2ee..b25f90e 100755 --- a/build/scripts/fetch-components.sh +++ b/build/scripts/fetch-components.sh @@ -51,6 +51,39 @@ if [ "$FETCH_ARCH" = "arm64" ]; then echo "==> Fetching RPi firmware..." "$SCRIPT_DIR/fetch-rpi-firmware.sh" + # Download ARM64 KubeSolo binary + KUBESOLO_VERSION="${KUBESOLO_VERSION:-v1.1.0}" + KUBESOLO_BIN_ARM64="$CACHE_DIR/kubesolo-arm64" + if [ -f "$KUBESOLO_BIN_ARM64" ]; then + echo "==> KubeSolo ARM64 binary already cached: $KUBESOLO_BIN_ARM64" + else + echo "==> Downloading KubeSolo ${KUBESOLO_VERSION} (arm64)..." + BIN_URL="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-linux-arm64-musl.tar.gz" + BIN_URL_FALLBACK="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-linux-arm64.tar.gz" + TEMP_DIR=$(mktemp -d) + echo " URL: $BIN_URL" + if curl -fSL "$BIN_URL" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then + echo " Downloaded musl variant (arm64)" + elif curl -fSL "$BIN_URL_FALLBACK" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then + echo " Downloaded glibc variant (arm64 fallback)" + else + echo "ERROR: Failed to download KubeSolo ARM64 from GitHub." + rm -rf "$TEMP_DIR" + exit 1 + fi + tar -xzf "$TEMP_DIR/kubesolo.tar.gz" -C "$TEMP_DIR" + FOUND_BIN=$(find "$TEMP_DIR" -name "kubesolo" -type f ! -name "*.tar.gz" | head -1) + if [ -z "$FOUND_BIN" ]; then + echo "ERROR: Could not find kubesolo binary in extracted archive" + rm -rf "$TEMP_DIR" + exit 1 + fi + cp "$FOUND_BIN" "$KUBESOLO_BIN_ARM64" + chmod +x "$KUBESOLO_BIN_ARM64" + rm -rf "$TEMP_DIR" + echo "==> KubeSolo ARM64 binary: $KUBESOLO_BIN_ARM64 ($(du -h "$KUBESOLO_BIN_ARM64" | cut -f1))" + fi + # Skip x86_64 ISO and TCZ downloads for ARM64 echo "" echo "==> ARM64 fetch complete." diff --git a/build/scripts/inject-kubesolo.sh b/build/scripts/inject-kubesolo.sh index 813c9ee..63a5d1c 100755 --- a/build/scripts/inject-kubesolo.sh +++ b/build/scripts/inject-kubesolo.sh @@ -25,7 +25,11 @@ if [ ! -d "$ROOTFS" ]; then exit 1 fi -KUBESOLO_BIN="$CACHE_DIR/kubesolo" +if [ "$INJECT_ARCH" = "arm64" ]; then + KUBESOLO_BIN="$CACHE_DIR/kubesolo-arm64" +else + KUBESOLO_BIN="$CACHE_DIR/kubesolo" +fi if [ ! -f "$KUBESOLO_BIN" ]; then echo "ERROR: KubeSolo binary not found: $KUBESOLO_BIN" echo "See fetch-components.sh output for instructions." @@ -78,23 +82,27 @@ for lib in network.sh health.sh; do done # Cloud-init binary (Go, built separately) -CLOUDINIT_BIN="$CACHE_DIR/kubesolo-cloudinit" +# Try arch-specific binary first, then fall back to generic +CLOUDINIT_BIN="$CACHE_DIR/kubesolo-cloudinit-linux-$INJECT_ARCH" +[ ! -f "$CLOUDINIT_BIN" ] && CLOUDINIT_BIN="$CACHE_DIR/kubesolo-cloudinit" if [ -f "$CLOUDINIT_BIN" ]; then cp "$CLOUDINIT_BIN" "$ROOTFS/usr/lib/kubesolo-os/kubesolo-cloudinit" chmod +x "$ROOTFS/usr/lib/kubesolo-os/kubesolo-cloudinit" echo " Installed cloud-init binary ($(du -h "$CLOUDINIT_BIN" | cut -f1))" else - echo " WARN: Cloud-init binary not found (run 'make build-cloudinit' to build)" + echo " WARN: Cloud-init binary not found (run 'make build-cloudinit' or 'make build-cross' to build)" fi # Update agent binary (Go, built separately) -UPDATE_BIN="$CACHE_DIR/kubesolo-update" +# Try arch-specific binary first, then fall back to generic +UPDATE_BIN="$CACHE_DIR/kubesolo-update-linux-$INJECT_ARCH" +[ ! -f "$UPDATE_BIN" ] && UPDATE_BIN="$CACHE_DIR/kubesolo-update" if [ -f "$UPDATE_BIN" ]; then cp "$UPDATE_BIN" "$ROOTFS/usr/lib/kubesolo-os/kubesolo-update" chmod +x "$ROOTFS/usr/lib/kubesolo-os/kubesolo-update" echo " Installed update agent ($(du -h "$UPDATE_BIN" | cut -f1))" else - echo " WARN: Update agent not found (run 'make build-update-agent' to build)" + echo " WARN: Update agent not found (run 'make build-update-agent' or 'make build-cross' to build)" fi # --- 3. Custom kernel or TCZ kernel modules --- @@ -115,8 +123,16 @@ for d in "$ROOTFS"/lib/modules/*/; do [ -d "$d" ] && KVER="$(basename "$d")" && break done +# Fallback: detect from custom kernel modules directory +if [ -z "$KVER" ] && [ -d "$CUSTOM_MODULES/lib/modules" ]; then + for d in "$CUSTOM_MODULES"/lib/modules/*/; do + [ -d "$d" ] && KVER="$(basename "$d")" && break + done + echo " Detected kernel version from custom kernel: $KVER" +fi + if [ -z "$KVER" ]; then - echo " WARN: Could not detect kernel version from rootfs" + echo " WARN: Could not detect kernel version from rootfs or custom kernel" fi echo " Kernel version: $KVER" @@ -145,24 +161,49 @@ if [ -f "$CUSTOM_VMLINUZ" ] && [ -d "$CUSTOM_MODULES/lib/modules/$KVER" ]; then [ -f "$CUSTOM_MOD_DIR/$f" ] && cp "$CUSTOM_MOD_DIR/$f" "$ROOTFS/lib/modules/$KVER/" done - # Use modprobe --show-depends to resolve each module + its transitive deps + # Resolve and install modules from modules.list + transitive deps if [ "$INJECT_ARCH" = "arm64" ]; then MODULES_LIST="$PROJECT_ROOT/build/config/modules-arm64.list" else MODULES_LIST="$PROJECT_ROOT/build/config/modules.list" fi NEEDED_MODS=$(mktemp) + + # Try modprobe first (works for same-arch builds) + MODPROBE_WORKS=true + FIRST_MOD=$(grep -v '^#' "$MODULES_LIST" | grep -v '^$' | head -1 | xargs) + if ! modprobe -S "$KVER" -d "$CUSTOM_MODULES" --show-depends "$FIRST_MOD" >/dev/null 2>&1; then + MODPROBE_WORKS=false + echo " modprobe cannot resolve modules (cross-arch build) — using find fallback" + fi + while IFS= read -r mod; do # Skip comments and blank lines case "$mod" in \#*|"") continue ;; esac mod=$(echo "$mod" | xargs) # trim whitespace [ -z "$mod" ] && continue - # modprobe -S -d --show-depends lists all deps in load order - # Output format: "insmod /path/to/module.ko" — extract path with awk - modprobe -S "$KVER" -d "$CUSTOM_MODULES" --show-depends "$mod" 2>/dev/null \ - | awk '/^insmod/{print $2}' >> "$NEEDED_MODS" \ - || echo " WARN: modprobe could not resolve: $mod" + if [ "$MODPROBE_WORKS" = true ]; then + # modprobe -S -d --show-depends lists all deps in load order + modprobe -S "$KVER" -d "$CUSTOM_MODULES" --show-depends "$mod" 2>/dev/null \ + | awk '/^insmod/{print $2}' >> "$NEEDED_MODS" \ + || echo " WARN: modprobe could not resolve: $mod" + else + # Cross-arch fallback: find module by name in kernel tree + found=$(find "$CUSTOM_MOD_DIR/kernel" -name "${mod}.ko" -o -name "${mod}.ko.xz" -o -name "${mod}.ko.gz" -o -name "${mod}.ko.zst" 2>/dev/null | head -1) + if [ -n "$found" ]; then + echo "$found" >> "$NEEDED_MODS" + else + # Try replacing hyphens with underscores and vice versa + mod_alt=$(echo "$mod" | tr '-' '_') + found=$(find "$CUSTOM_MOD_DIR/kernel" -name "${mod_alt}.ko" -o -name "${mod_alt}.ko.xz" -o -name "${mod_alt}.ko.gz" -o -name "${mod_alt}.ko.zst" 2>/dev/null | head -1) + if [ -n "$found" ]; then + echo "$found" >> "$NEEDED_MODS" + else + echo " WARN: could not find module: $mod" + fi + fi + fi done < "$MODULES_LIST" # Deduplicate and copy each needed module