feat: custom kernel build + boot fixes for working container runtime
Build a custom Tiny Core 17.0 kernel (6.18.2) with missing configs that the stock kernel lacks for container workloads: - CONFIG_CGROUP_BPF=y (cgroup v2 device control via BPF) - CONFIG_DEVTMPFS=y (auto-create /dev device nodes) - CONFIG_DEVTMPFS_MOUNT=y (auto-mount devtmpfs) - CONFIG_MEMCG=y (memory cgroup controller for memory.max) - CONFIG_CFS_BANDWIDTH=y (CPU bandwidth throttling for cpu.max) Also strips unnecessary subsystems (sound, GPU, wireless, Bluetooth, KVM, etc.) for minimal footprint on a headless K8s edge appliance. Init system fixes for successful boot-to-running-pods: - Add switch_root in init.sh to escape initramfs (runc pivot_root) - Add mountpoint guards in 00-early-mount.sh (skip if already mounted) - Create essential device nodes after switch_root (kmsg, console, etc.) - Enable cgroup v2 controller delegation with init process isolation - Mount BPF filesystem for cgroup v2 device control - Add mknod fallback from sysfs in 20-persistent-mount.sh for /dev/vda - Move KubeSolo binary to /usr/bin (avoid /usr/local bind mount hiding) - Generate /etc/machine-id in 60-hostname.sh (kubelet requires it) - Pre-initialize iptables tables before kube-proxy starts - Add nft_reject, nft_fib, xt_nfacct to kernel modules list Build system changes: - New build-kernel.sh script for custom kernel compilation - Dockerfile.builder adds kernel build deps (flex, bison, libelf, etc.) - Selective kernel module install (only modules.list + transitive deps) - Install iptables-nft (xtables-nft-multi) + shared libs in rootfs Tested: ISO boots in QEMU, node reaches Ready in ~35s, CoreDNS and local-path-provisioner pods start and run successfully. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
209
build/scripts/build-kernel.sh
Executable file
209
build/scripts/build-kernel.sh
Executable file
@@ -0,0 +1,209 @@
|
||||
#!/bin/bash
|
||||
# build-kernel.sh — Build custom Tiny Core kernel with CONFIG_CGROUP_BPF=y
|
||||
#
|
||||
# The stock Tiny Core 17.0 kernel (6.18.2-tinycore64) lacks CONFIG_CGROUP_BPF,
|
||||
# which is required for cgroup v2 device control in runc/containerd.
|
||||
# This script downloads the TC-patched kernel source, enables CONFIG_CGROUP_BPF,
|
||||
# and builds vmlinuz + modules.
|
||||
#
|
||||
# Output is cached in $CACHE_DIR/custom-kernel/ and reused across builds.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
||||
|
||||
# shellcheck source=../config/versions.env
|
||||
. "$SCRIPT_DIR/../config/versions.env"
|
||||
|
||||
KVER="6.18.2-tinycore64"
|
||||
KERNEL_BASE_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/release/src/kernel"
|
||||
KERNEL_SRC_URL="${KERNEL_BASE_URL}/linux-6.18.2-patched.tar.xz"
|
||||
KERNEL_CFG_URL="${KERNEL_BASE_URL}/config-${KVER}"
|
||||
|
||||
CUSTOM_KERNEL_DIR="$CACHE_DIR/custom-kernel"
|
||||
CUSTOM_VMLINUZ="$CUSTOM_KERNEL_DIR/vmlinuz"
|
||||
CUSTOM_MODULES="$CUSTOM_KERNEL_DIR/modules"
|
||||
|
||||
mkdir -p "$CACHE_DIR" "$CUSTOM_KERNEL_DIR"
|
||||
|
||||
# --- Skip if already built ---
|
||||
if [ -f "$CUSTOM_VMLINUZ" ] && [ -d "$CUSTOM_MODULES/lib/modules/$KVER" ]; then
|
||||
echo "==> Custom kernel already built (cached)"
|
||||
echo " vmlinuz: $CUSTOM_VMLINUZ ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||
MOD_COUNT=$(find "$CUSTOM_MODULES/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||
echo " Modules: $MOD_COUNT modules in $CUSTOM_MODULES/lib/modules/$KVER"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "==> Building custom kernel with CONFIG_CGROUP_BPF=y..."
|
||||
echo " Kernel version: $KVER"
|
||||
|
||||
# --- Download kernel source ---
|
||||
KERNEL_SRC_ARCHIVE="$CACHE_DIR/linux-6.18.2-patched.tar.xz"
|
||||
if [ ! -f "$KERNEL_SRC_ARCHIVE" ]; then
|
||||
echo "==> Downloading kernel source (~149 MB)..."
|
||||
echo " URL: $KERNEL_SRC_URL"
|
||||
wget -q --show-progress -O "$KERNEL_SRC_ARCHIVE" "$KERNEL_SRC_URL" 2>/dev/null || \
|
||||
curl -fSL "$KERNEL_SRC_URL" -o "$KERNEL_SRC_ARCHIVE"
|
||||
echo " Downloaded: $(du -h "$KERNEL_SRC_ARCHIVE" | cut -f1)"
|
||||
else
|
||||
echo "==> Kernel source already cached: $(du -h "$KERNEL_SRC_ARCHIVE" | cut -f1)"
|
||||
fi
|
||||
|
||||
# --- Download stock config ---
|
||||
KERNEL_CFG="$CACHE_DIR/config-${KVER}"
|
||||
if [ ! -f "$KERNEL_CFG" ]; then
|
||||
echo "==> Downloading stock kernel config..."
|
||||
echo " URL: $KERNEL_CFG_URL"
|
||||
wget -q -O "$KERNEL_CFG" "$KERNEL_CFG_URL" 2>/dev/null || \
|
||||
curl -fSL "$KERNEL_CFG_URL" -o "$KERNEL_CFG"
|
||||
else
|
||||
echo "==> Stock kernel config already cached"
|
||||
fi
|
||||
|
||||
# --- Extract source ---
|
||||
# IMPORTANT: Must extract on a case-sensitive filesystem. The kernel source has
|
||||
# files that differ only by case (e.g., xt_mark.h vs xt_MARK.h). If the cache
|
||||
# is on macOS (case-insensitive APFS), extraction silently loses files.
|
||||
# Use /tmp inside the container (ext4, case-sensitive) for the build.
|
||||
KERNEL_BUILD_DIR="/tmp/kernel-build"
|
||||
rm -rf "$KERNEL_BUILD_DIR"
|
||||
mkdir -p "$KERNEL_BUILD_DIR"
|
||||
|
||||
echo "==> Extracting kernel source (case-sensitive filesystem)..."
|
||||
tar -xf "$KERNEL_SRC_ARCHIVE" -C "$KERNEL_BUILD_DIR"
|
||||
|
||||
# Find the extracted source directory (could be linux-6.18.2 or linux-6.18.2-patched)
|
||||
KERNEL_SRC_DIR=$(find "$KERNEL_BUILD_DIR" -maxdepth 1 -type d -name 'linux-*' | head -1)
|
||||
if [ -z "$KERNEL_SRC_DIR" ]; then
|
||||
echo "ERROR: Could not find kernel source directory after extraction"
|
||||
ls -la "$KERNEL_BUILD_DIR"/
|
||||
exit 1
|
||||
fi
|
||||
echo " Source dir: $(basename "$KERNEL_SRC_DIR")"
|
||||
|
||||
cd "$KERNEL_SRC_DIR"
|
||||
|
||||
# --- Apply stock config + enable CONFIG_CGROUP_BPF ---
|
||||
echo "==> Applying stock Tiny Core config..."
|
||||
cp "$KERNEL_CFG" .config
|
||||
|
||||
echo "==> Enabling required kernel configs..."
|
||||
./scripts/config --enable CONFIG_CGROUP_BPF
|
||||
./scripts/config --enable CONFIG_DEVTMPFS
|
||||
./scripts/config --enable CONFIG_DEVTMPFS_MOUNT
|
||||
./scripts/config --enable CONFIG_MEMCG
|
||||
./scripts/config --enable CONFIG_CFS_BANDWIDTH
|
||||
|
||||
# --- Strip unnecessary subsystems for smallest footprint ---
|
||||
# This is a headless K8s edge appliance — no sound, GPU, wireless, etc.
|
||||
echo "==> Disabling unnecessary subsystems for minimal footprint..."
|
||||
|
||||
# Sound subsystem (not needed on headless appliance)
|
||||
./scripts/config --disable SOUND
|
||||
|
||||
# GPU/DRM (serial console only, no display)
|
||||
./scripts/config --disable DRM
|
||||
|
||||
# KVM hypervisor (this IS the guest/bare metal, not a hypervisor)
|
||||
./scripts/config --disable KVM
|
||||
|
||||
# Media/camera/TV/radio (not needed)
|
||||
./scripts/config --disable MEDIA_SUPPORT
|
||||
|
||||
# Wireless networking (wired edge device)
|
||||
./scripts/config --disable WIRELESS
|
||||
./scripts/config --disable WLAN
|
||||
./scripts/config --disable CFG80211
|
||||
|
||||
# Bluetooth (not needed)
|
||||
./scripts/config --disable BT
|
||||
|
||||
# NFC (not needed)
|
||||
./scripts/config --disable NFC
|
||||
|
||||
# Infiniband (not needed on edge)
|
||||
./scripts/config --disable INFINIBAND
|
||||
|
||||
# PCMCIA (legacy, not needed)
|
||||
./scripts/config --disable PCMCIA
|
||||
|
||||
# Amateur radio (not needed)
|
||||
./scripts/config --disable HAMRADIO
|
||||
|
||||
# ISDN (not needed)
|
||||
./scripts/config --disable ISDN
|
||||
|
||||
# ATM networking (not needed)
|
||||
./scripts/config --disable ATM
|
||||
|
||||
# Joystick/gamepad (not needed)
|
||||
./scripts/config --disable INPUT_JOYSTICK
|
||||
./scripts/config --disable INPUT_TABLET
|
||||
|
||||
# FPGA (not needed)
|
||||
./scripts/config --disable FPGA
|
||||
|
||||
# Resolve dependencies (olddefconfig accepts defaults for new options)
|
||||
make olddefconfig
|
||||
|
||||
# Verify CONFIG_CGROUP_BPF is set
|
||||
if grep -q 'CONFIG_CGROUP_BPF=y' .config; then
|
||||
echo " CONFIG_CGROUP_BPF=y confirmed in .config"
|
||||
else
|
||||
echo "ERROR: CONFIG_CGROUP_BPF not set after olddefconfig"
|
||||
grep 'CGROUP_BPF' .config || echo " (CGROUP_BPF not found in .config)"
|
||||
echo ""
|
||||
echo "Prerequisites check:"
|
||||
grep -E 'CONFIG_BPF=|CONFIG_BPF_SYSCALL=' .config || echo " BPF not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Show what changed
|
||||
echo " Config diff from stock:"
|
||||
diff "$KERNEL_CFG" .config | grep '^[<>]' | head -20 || echo " (no differences beyond CGROUP_BPF)"
|
||||
|
||||
# --- Build kernel + modules ---
|
||||
NPROC=$(nproc 2>/dev/null || echo 4)
|
||||
echo ""
|
||||
echo "==> Building kernel (${NPROC} parallel jobs)..."
|
||||
echo " This may take 15-25 minutes..."
|
||||
|
||||
make -j"$NPROC" bzImage modules 2>&1
|
||||
|
||||
echo "==> Kernel build complete"
|
||||
|
||||
# --- Install to staging ---
|
||||
echo "==> Installing vmlinuz..."
|
||||
cp arch/x86/boot/bzImage "$CUSTOM_VMLINUZ"
|
||||
|
||||
echo "==> Installing modules (stripped)..."
|
||||
rm -rf "$CUSTOM_MODULES"
|
||||
mkdir -p "$CUSTOM_MODULES"
|
||||
make INSTALL_MOD_STRIP=1 modules_install INSTALL_MOD_PATH="$CUSTOM_MODULES"
|
||||
|
||||
# Remove build/source symlinks (they point to the build dir which won't exist in rootfs)
|
||||
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/build"
|
||||
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/source"
|
||||
|
||||
# Run depmod to generate proper module dependency files
|
||||
echo "==> Running depmod..."
|
||||
depmod -a -b "$CUSTOM_MODULES" "$KVER" 2>/dev/null || true
|
||||
|
||||
# Save the final config for reference
|
||||
cp .config "$CUSTOM_KERNEL_DIR/.config"
|
||||
|
||||
# --- Clean up build dir (large, ~1.5 GB) ---
|
||||
echo "==> Cleaning kernel build directory..."
|
||||
cd /
|
||||
rm -rf "$KERNEL_BUILD_DIR"
|
||||
|
||||
# --- Summary ---
|
||||
echo ""
|
||||
echo "==> Custom kernel build complete:"
|
||||
echo " vmlinuz: $CUSTOM_VMLINUZ ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||
MOD_COUNT=$(find "$CUSTOM_MODULES/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||
echo " Modules: $MOD_COUNT modules"
|
||||
echo " Modules size: $(du -sh "$CUSTOM_MODULES/lib/modules/$KVER" | cut -f1)"
|
||||
echo ""
|
||||
@@ -31,36 +31,126 @@ else
|
||||
fi
|
||||
|
||||
# --- KubeSolo ---
|
||||
KUBESOLO_INSTALLER="$CACHE_DIR/install-kubesolo.sh"
|
||||
KUBESOLO_VERSION="${KUBESOLO_VERSION:-v1.1.0}"
|
||||
KUBESOLO_BIN="$CACHE_DIR/kubesolo"
|
||||
|
||||
if [ -f "$KUBESOLO_BIN" ]; then
|
||||
echo "==> KubeSolo binary already cached: $KUBESOLO_BIN"
|
||||
else
|
||||
echo "==> Downloading KubeSolo installer..."
|
||||
curl -sfL "$KUBESOLO_INSTALL_URL" -o "$KUBESOLO_INSTALLER"
|
||||
echo "==> Downloading KubeSolo ${KUBESOLO_VERSION}..."
|
||||
|
||||
echo "==> Extracting KubeSolo binary..."
|
||||
echo " NOTE: The installer normally runs 'install'. We extract the binary URL instead."
|
||||
echo " For Phase 1 PoC, install KubeSolo on a host and copy the binary."
|
||||
echo ""
|
||||
echo " Manual step required:"
|
||||
echo " 1. On a Linux x86_64 host: curl -sfL https://get.kubesolo.io | sudo sh -"
|
||||
echo " 2. Copy /usr/local/bin/kubesolo to: $KUBESOLO_BIN"
|
||||
echo " 3. Re-run: make rootfs"
|
||||
echo ""
|
||||
# Determine architecture
|
||||
ARCH="${TARGET_ARCH:-amd64}"
|
||||
OS="linux"
|
||||
|
||||
# Try to extract download URL from installer script
|
||||
BINARY_URL=$(grep -oP 'https://[^ ]+kubesolo[^ ]+' "$KUBESOLO_INSTALLER" 2>/dev/null | head -1 || true)
|
||||
if [ -n "$BINARY_URL" ]; then
|
||||
echo " Attempting direct download from: $BINARY_URL"
|
||||
curl -sfL "$BINARY_URL" -o "$KUBESOLO_BIN" && chmod +x "$KUBESOLO_BIN" || {
|
||||
echo " Direct download failed. Use manual step above."
|
||||
}
|
||||
# Build download URL from GitHub releases
|
||||
# Available variants: kubesolo-v1.1.0-linux-amd64.tar.gz, kubesolo-v1.1.0-linux-amd64-musl.tar.gz
|
||||
# We use the musl variant for maximum compatibility with Tiny Core Linux (musl-based)
|
||||
BIN_URL="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-${OS}-${ARCH}-musl.tar.gz"
|
||||
BIN_URL_FALLBACK="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-${OS}-${ARCH}.tar.gz"
|
||||
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf '$TEMP_DIR'" EXIT
|
||||
|
||||
echo " URL: $BIN_URL"
|
||||
if curl -fSL "$BIN_URL" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||
echo " Downloaded musl variant"
|
||||
elif curl -fSL "$BIN_URL_FALLBACK" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||
echo " Downloaded glibc variant (fallback)"
|
||||
else
|
||||
echo "ERROR: Failed to download KubeSolo from GitHub."
|
||||
echo " Tried: $BIN_URL"
|
||||
echo " Tried: $BIN_URL_FALLBACK"
|
||||
echo ""
|
||||
echo " Manual step:"
|
||||
echo " 1. Download from: https://github.com/portainer/kubesolo/releases"
|
||||
echo " 2. Extract and copy binary to: $KUBESOLO_BIN"
|
||||
echo " 3. Re-run: make rootfs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -f "$KUBESOLO_BIN" ]; then
|
||||
echo "==> KubeSolo binary: $KUBESOLO_BIN ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
||||
# Extract binary from tarball
|
||||
tar -xzf "$TEMP_DIR/kubesolo.tar.gz" -C "$TEMP_DIR"
|
||||
|
||||
# Find the kubesolo binary in extracted contents
|
||||
FOUND_BIN=$(find "$TEMP_DIR" -name "kubesolo" -type f ! -name "*.tar.gz" | head -1)
|
||||
if [ -z "$FOUND_BIN" ]; then
|
||||
echo "ERROR: Could not find kubesolo binary in extracted archive"
|
||||
echo " Archive contents:"
|
||||
ls -la "$TEMP_DIR"/
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cp "$FOUND_BIN" "$KUBESOLO_BIN"
|
||||
chmod +x "$KUBESOLO_BIN"
|
||||
|
||||
trap - EXIT
|
||||
rm -rf "$TEMP_DIR"
|
||||
|
||||
echo "==> KubeSolo binary: $KUBESOLO_BIN ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
||||
fi
|
||||
|
||||
# --- Tiny Core kernel module extensions (netfilter, iptables) ---
|
||||
# The base Tiny Core initramfs does NOT include netfilter kernel modules.
|
||||
# They are distributed as separate TCZ (squashfs) extensions.
|
||||
# KubeSolo requires netfilter for kube-proxy, iptables NAT, conntrack, etc.
|
||||
|
||||
# Detect kernel version from the cached ISO
|
||||
KVER=""
|
||||
if [ -f "$TC_ISO" ]; then
|
||||
# Try to detect kernel version from ISO without mounting
|
||||
# Tiny Core 17.0 uses 6.18.2-tinycore64
|
||||
KVER="6.18.2-tinycore64"
|
||||
fi
|
||||
|
||||
NETFILTER_TCZ="$CACHE_DIR/ipv6-netfilter-${KVER}.tcz"
|
||||
NETFILTER_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/ipv6-netfilter-${KVER}.tcz"
|
||||
|
||||
if [ -f "$NETFILTER_TCZ" ]; then
|
||||
echo "==> Netfilter modules already cached: $NETFILTER_TCZ"
|
||||
else
|
||||
echo "==> Downloading netfilter kernel modules (ipv6-netfilter-${KVER}.tcz)..."
|
||||
echo " URL: $NETFILTER_TCZ_URL"
|
||||
if wget -q --show-progress -O "$NETFILTER_TCZ" "$NETFILTER_TCZ_URL" 2>/dev/null || \
|
||||
curl -fSL "$NETFILTER_TCZ_URL" -o "$NETFILTER_TCZ" 2>/dev/null; then
|
||||
echo "==> Downloaded: $NETFILTER_TCZ ($(du -h "$NETFILTER_TCZ" | cut -f1))"
|
||||
else
|
||||
echo "WARN: Failed to download netfilter modules. kube-proxy may not work."
|
||||
rm -f "$NETFILTER_TCZ"
|
||||
fi
|
||||
fi
|
||||
|
||||
NET_BRIDGING_TCZ="$CACHE_DIR/net-bridging-${KVER}.tcz"
|
||||
NET_BRIDGING_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/net-bridging-${KVER}.tcz"
|
||||
|
||||
if [ -f "$NET_BRIDGING_TCZ" ]; then
|
||||
echo "==> Net-bridging modules already cached: $NET_BRIDGING_TCZ"
|
||||
else
|
||||
echo "==> Downloading net-bridging kernel modules (net-bridging-${KVER}.tcz)..."
|
||||
echo " URL: $NET_BRIDGING_TCZ_URL"
|
||||
if wget -q --show-progress -O "$NET_BRIDGING_TCZ" "$NET_BRIDGING_TCZ_URL" 2>/dev/null || \
|
||||
curl -fSL "$NET_BRIDGING_TCZ_URL" -o "$NET_BRIDGING_TCZ" 2>/dev/null; then
|
||||
echo "==> Downloaded: $NET_BRIDGING_TCZ ($(du -h "$NET_BRIDGING_TCZ" | cut -f1))"
|
||||
else
|
||||
echo "WARN: Failed to download net-bridging modules. CNI bridge may not work."
|
||||
rm -f "$NET_BRIDGING_TCZ"
|
||||
fi
|
||||
fi
|
||||
|
||||
IPTABLES_TCZ="$CACHE_DIR/iptables.tcz"
|
||||
IPTABLES_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/iptables.tcz"
|
||||
|
||||
if [ -f "$IPTABLES_TCZ" ]; then
|
||||
echo "==> iptables userspace already cached: $IPTABLES_TCZ"
|
||||
else
|
||||
echo "==> Downloading iptables userspace tools..."
|
||||
echo " URL: $IPTABLES_TCZ_URL"
|
||||
if wget -q --show-progress -O "$IPTABLES_TCZ" "$IPTABLES_TCZ_URL" 2>/dev/null || \
|
||||
curl -fSL "$IPTABLES_TCZ_URL" -o "$IPTABLES_TCZ" 2>/dev/null; then
|
||||
echo "==> Downloaded: $IPTABLES_TCZ ($(du -h "$IPTABLES_TCZ" | cut -f1))"
|
||||
else
|
||||
echo "WARN: Failed to download iptables. KubeSolo bundles its own but this is a fallback."
|
||||
rm -f "$IPTABLES_TCZ"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -25,15 +25,19 @@ fi
|
||||
echo "==> Injecting KubeSolo into rootfs..."
|
||||
|
||||
# --- 1. KubeSolo binary ---
|
||||
mkdir -p "$ROOTFS/usr/local/bin"
|
||||
cp "$KUBESOLO_BIN" "$ROOTFS/usr/local/bin/kubesolo"
|
||||
chmod +x "$ROOTFS/usr/local/bin/kubesolo"
|
||||
# Install to /usr/bin (NOT /usr/local/bin) because /usr/local is bind-mounted
|
||||
# from the data partition at boot, which would hide the binary.
|
||||
mkdir -p "$ROOTFS/usr/bin"
|
||||
cp "$KUBESOLO_BIN" "$ROOTFS/usr/bin/kubesolo"
|
||||
chmod +x "$ROOTFS/usr/bin/kubesolo"
|
||||
echo " Installed KubeSolo binary ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
||||
|
||||
# --- 2. Custom init system ---
|
||||
echo " Installing init system..."
|
||||
|
||||
# Main init
|
||||
# Main init — remove symlink first to avoid clobbering busybox
|
||||
# (Tiny Core has /sbin/init -> ../bin/busybox; cp follows symlinks)
|
||||
rm -f "$ROOTFS/sbin/init"
|
||||
cp "$PROJECT_ROOT/init/init.sh" "$ROOTFS/sbin/init"
|
||||
chmod +x "$ROOTFS/sbin/init"
|
||||
|
||||
@@ -83,7 +87,233 @@ else
|
||||
echo " WARN: Update agent not found (run 'make build-update-agent' to build)"
|
||||
fi
|
||||
|
||||
# --- 3. Kernel modules list ---
|
||||
# --- 3. Custom kernel or TCZ kernel modules ---
|
||||
# If a custom kernel was built (with CONFIG_CGROUP_BPF=y), use it.
|
||||
# Otherwise fall back to TCZ-extracted modules with manual modules.dep.
|
||||
CUSTOM_KERNEL_DIR="$CACHE_DIR/custom-kernel"
|
||||
CUSTOM_VMLINUZ="$CUSTOM_KERNEL_DIR/vmlinuz"
|
||||
CUSTOM_MODULES="$CUSTOM_KERNEL_DIR/modules"
|
||||
|
||||
# Detect kernel version from rootfs
|
||||
KVER=""
|
||||
for d in "$ROOTFS"/lib/modules/*/; do
|
||||
[ -d "$d" ] && KVER="$(basename "$d")" && break
|
||||
done
|
||||
|
||||
if [ -z "$KVER" ]; then
|
||||
echo " WARN: Could not detect kernel version from rootfs"
|
||||
fi
|
||||
|
||||
echo " Kernel version: $KVER"
|
||||
|
||||
if [ -f "$CUSTOM_VMLINUZ" ] && [ -d "$CUSTOM_MODULES/lib/modules/$KVER" ]; then
|
||||
# =========================================================================
|
||||
# Custom kernel path — selective module install (only what modules.list needs)
|
||||
# =========================================================================
|
||||
echo " Using custom kernel (CONFIG_CGROUP_BPF=y)..."
|
||||
|
||||
# Replace vmlinuz
|
||||
cp "$CUSTOM_VMLINUZ" "$ROOTFS_DIR/vmlinuz"
|
||||
echo " Installed custom vmlinuz ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||
|
||||
# Selectively install ONLY modules from modules.list + their transitive deps.
|
||||
# This keeps the initramfs minimal — no sound, GPU, SCSI, etc. modules.
|
||||
echo " Installing kernel modules (selective — modules.list + deps only)..."
|
||||
CUSTOM_MOD_DIR="$CUSTOM_MODULES/lib/modules/$KVER"
|
||||
|
||||
rm -rf "$ROOTFS/lib/modules/$KVER"
|
||||
mkdir -p "$ROOTFS/lib/modules/$KVER/kernel"
|
||||
|
||||
# Copy module metadata files (needed by modprobe)
|
||||
for f in modules.builtin modules.builtin.modinfo modules.order \
|
||||
modules.builtin.alias.bin modules.builtin.bin; do
|
||||
[ -f "$CUSTOM_MOD_DIR/$f" ] && cp "$CUSTOM_MOD_DIR/$f" "$ROOTFS/lib/modules/$KVER/"
|
||||
done
|
||||
|
||||
# Use modprobe --show-depends to resolve each module + its transitive deps
|
||||
MODULES_LIST="$PROJECT_ROOT/build/config/modules.list"
|
||||
NEEDED_MODS=$(mktemp)
|
||||
while IFS= read -r mod; do
|
||||
# Skip comments and blank lines
|
||||
case "$mod" in \#*|"") continue ;; esac
|
||||
mod=$(echo "$mod" | xargs) # trim whitespace
|
||||
[ -z "$mod" ] && continue
|
||||
|
||||
# modprobe -S <ver> -d <root> --show-depends <module> lists all deps in load order
|
||||
# Output format: "insmod /path/to/module.ko" — extract path with awk
|
||||
modprobe -S "$KVER" -d "$CUSTOM_MODULES" --show-depends "$mod" 2>/dev/null \
|
||||
| awk '/^insmod/{print $2}' >> "$NEEDED_MODS" \
|
||||
|| echo " WARN: modprobe could not resolve: $mod"
|
||||
done < "$MODULES_LIST"
|
||||
|
||||
# Deduplicate and copy each needed module
|
||||
sort -u "$NEEDED_MODS" | while IFS= read -r mod_path; do
|
||||
mod_path=$(echo "$mod_path" | xargs) # trim whitespace
|
||||
[ -z "$mod_path" ] && continue
|
||||
# mod_path is absolute (e.g., /path/to/custom-kernel/modules/lib/modules/KVER/kernel/...)
|
||||
if [ ! -f "$mod_path" ]; then
|
||||
echo " WARN: module not found: $mod_path"
|
||||
continue
|
||||
fi
|
||||
# Get the relative path under lib/modules/KVER/
|
||||
rel_path="${mod_path#$CUSTOM_MOD_DIR/}"
|
||||
dst="$ROOTFS/lib/modules/$KVER/$rel_path"
|
||||
mkdir -p "$(dirname "$dst")"
|
||||
cp "$mod_path" "$dst"
|
||||
done
|
||||
rm -f "$NEEDED_MODS"
|
||||
|
||||
# Run depmod on the selective module set to generate correct metadata
|
||||
depmod -a -b "$ROOTFS" "$KVER" 2>/dev/null || true
|
||||
|
||||
MOD_COUNT=$(find "$ROOTFS/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||
MOD_SIZE=$(du -sh "$ROOTFS/lib/modules/$KVER" | cut -f1)
|
||||
echo " Installed $MOD_COUNT kernel modules ($MOD_SIZE) — minimal set"
|
||||
|
||||
else
|
||||
# =========================================================================
|
||||
# Stock kernel path — extract TCZ modules + manual modules.dep
|
||||
# =========================================================================
|
||||
echo " No custom kernel found, using stock kernel with TCZ modules..."
|
||||
|
||||
if [ -n "$KVER" ]; then
|
||||
ROOTFS_MOD_DST="$ROOTFS/lib/modules/$KVER/kernel"
|
||||
|
||||
NETFILTER_TCZ="$CACHE_DIR/ipv6-netfilter-${KVER}.tcz"
|
||||
if [ -f "$NETFILTER_TCZ" ]; then
|
||||
echo " Extracting netfilter modules from $(basename "$NETFILTER_TCZ")..."
|
||||
TCZ_TMP=$(mktemp -d)
|
||||
if command -v unsquashfs >/dev/null 2>&1; then
|
||||
unsquashfs -d "$TCZ_TMP/content" "$NETFILTER_TCZ" >/dev/null 2>&1
|
||||
else
|
||||
echo " ERROR: unsquashfs not found (install squashfs-tools)"
|
||||
rm -rf "$TCZ_TMP"
|
||||
exit 1
|
||||
fi
|
||||
TCZ_MOD_SRC="$TCZ_TMP/content/usr/local/lib/modules/$KVER/kernel"
|
||||
if [ -d "$TCZ_MOD_SRC" ]; then
|
||||
find "$TCZ_MOD_SRC" -name '*.ko.gz' | while IFS= read -r mod_file; do
|
||||
rel_path="${mod_file#$TCZ_MOD_SRC/}"
|
||||
dst_dir="$ROOTFS_MOD_DST/$(dirname "$rel_path")"
|
||||
mkdir -p "$dst_dir"
|
||||
cp "$mod_file" "$dst_dir/"
|
||||
done
|
||||
MOD_COUNT=$(find "$TCZ_MOD_SRC" -name '*.ko.gz' | wc -l)
|
||||
echo " Installed $MOD_COUNT kernel modules from netfilter TCZ"
|
||||
fi
|
||||
rm -rf "$TCZ_TMP"
|
||||
else
|
||||
echo " WARN: Netfilter TCZ not found. kube-proxy may not work."
|
||||
fi
|
||||
|
||||
NET_BRIDGING_TCZ="$CACHE_DIR/net-bridging-${KVER}.tcz"
|
||||
if [ -f "$NET_BRIDGING_TCZ" ]; then
|
||||
echo " Extracting bridge modules from $(basename "$NET_BRIDGING_TCZ")..."
|
||||
TCZ_TMP=$(mktemp -d)
|
||||
unsquashfs -d "$TCZ_TMP/content" "$NET_BRIDGING_TCZ" >/dev/null 2>&1
|
||||
TCZ_MOD_SRC="$TCZ_TMP/content/usr/local/lib/modules/$KVER/kernel"
|
||||
if [ -d "$TCZ_MOD_SRC" ]; then
|
||||
find "$TCZ_MOD_SRC" -name '*.ko.gz' | while IFS= read -r mod_file; do
|
||||
rel_path="${mod_file#$TCZ_MOD_SRC/}"
|
||||
dst_dir="$ROOTFS_MOD_DST/$(dirname "$rel_path")"
|
||||
mkdir -p "$dst_dir"
|
||||
cp "$mod_file" "$dst_dir/"
|
||||
done
|
||||
BR_COUNT=$(find "$TCZ_MOD_SRC" -name '*.ko.gz' | wc -l)
|
||||
echo " Installed $BR_COUNT kernel modules from net-bridging TCZ"
|
||||
fi
|
||||
rm -rf "$TCZ_TMP"
|
||||
else
|
||||
echo " WARN: Net-bridging TCZ not found. CNI bridge networking may not work."
|
||||
fi
|
||||
|
||||
# Manual modules.dep for stock kernel (Ubuntu's depmod can't handle TC's kernel)
|
||||
MODULES_DEP="$ROOTFS/lib/modules/$KVER/modules.dep"
|
||||
if [ -f "$MODULES_DEP" ]; then
|
||||
echo " Appending module entries to modules.dep..."
|
||||
cat >> "$MODULES_DEP" << 'MODDEP'
|
||||
kernel/net/ipv6/ipv6.ko.gz:
|
||||
kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko.gz:
|
||||
kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko.gz: kernel/net/ipv6/ipv6.ko.gz
|
||||
kernel/net/netfilter/nf_conntrack.ko.gz: kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko.gz kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko.gz
|
||||
kernel/net/netfilter/nf_nat.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/nf_conntrack_netlink.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/nf_tables.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/nft_compat.ko.gz: kernel/net/netfilter/nf_tables.ko.gz
|
||||
kernel/net/netfilter/nft_chain_nat.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz
|
||||
kernel/net/netfilter/nft_ct.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/nft_masq.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/nft_nat.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz
|
||||
kernel/net/netfilter/nft_redir.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz
|
||||
kernel/net/netfilter/xt_conntrack.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/xt_MASQUERADE.ko.gz: kernel/net/netfilter/nf_nat.ko.gz kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/xt_mark.ko.gz:
|
||||
kernel/net/netfilter/xt_comment.ko.gz:
|
||||
kernel/net/netfilter/xt_multiport.ko.gz:
|
||||
kernel/net/netfilter/xt_nat.ko.gz: kernel/net/netfilter/nf_nat.ko.gz
|
||||
kernel/net/netfilter/xt_addrtype.ko.gz:
|
||||
kernel/net/netfilter/xt_connmark.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||
kernel/net/netfilter/xt_REDIRECT.ko.gz: kernel/net/netfilter/nf_nat.ko.gz
|
||||
kernel/net/netfilter/xt_recent.ko.gz:
|
||||
kernel/net/netfilter/xt_statistic.ko.gz:
|
||||
kernel/net/netfilter/xt_set.ko.gz: kernel/net/netfilter/ipset/ip_set.ko.gz
|
||||
kernel/net/netfilter/ipset/ip_set.ko.gz:
|
||||
kernel/net/ipv4/netfilter/nf_reject_ipv4.ko.gz:
|
||||
kernel/net/ipv6/netfilter/nf_reject_ipv6.ko.gz:
|
||||
kernel/net/ipv4/netfilter/ipt_REJECT.ko.gz: kernel/net/ipv4/netfilter/nf_reject_ipv4.ko.gz
|
||||
kernel/net/ipv6/netfilter/ip6t_REJECT.ko.gz: kernel/net/ipv6/netfilter/nf_reject_ipv6.ko.gz
|
||||
kernel/net/netfilter/nft_reject.ko.gz: kernel/net/netfilter/nf_tables.ko.gz
|
||||
kernel/net/bridge/bridge.ko.gz: kernel/net/802/stp.ko.gz kernel/net/llc/llc.ko.gz
|
||||
kernel/net/bridge/br_netfilter.ko.gz: kernel/net/bridge/bridge.ko.gz kernel/net/802/stp.ko.gz kernel/net/llc/llc.ko.gz
|
||||
kernel/net/bridge/netfilter/nf_conntrack_bridge.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz kernel/net/bridge/bridge.ko.gz
|
||||
MODDEP
|
||||
|
||||
find "$ROOTFS_MOD_DST" -name '*.ko.gz' -path '*/net/*' | sort | while IFS= read -r mod_file; do
|
||||
rel_path="kernel/${mod_file#$ROOTFS_MOD_DST/}"
|
||||
if ! grep -q "^${rel_path}:" "$MODULES_DEP" 2>/dev/null; then
|
||||
echo "${rel_path}:" >> "$MODULES_DEP"
|
||||
fi
|
||||
done
|
||||
echo " Updated modules.dep with netfilter entries"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Install iptables-nft (nftables-based iptables) from the builder system
|
||||
# Kernel 6.18 uses nf_tables, not legacy ip_tables, so we need xtables-nft-multi
|
||||
echo " Installing iptables-nft from builder..."
|
||||
if [ -f /usr/sbin/xtables-nft-multi ]; then
|
||||
mkdir -p "$ROOTFS/usr/sbin"
|
||||
cp /usr/sbin/xtables-nft-multi "$ROOTFS/usr/sbin/"
|
||||
|
||||
# Create standard symlinks
|
||||
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
|
||||
ln -sf xtables-nft-multi "$ROOTFS/usr/sbin/$cmd"
|
||||
done
|
||||
|
||||
# Copy required shared libraries
|
||||
mkdir -p "$ROOTFS/usr/lib/x86_64-linux-gnu" "$ROOTFS/lib/x86_64-linux-gnu" "$ROOTFS/lib64"
|
||||
for lib in \
|
||||
/lib/x86_64-linux-gnu/libxtables.so.12* \
|
||||
/lib/x86_64-linux-gnu/libmnl.so.0* \
|
||||
/lib/x86_64-linux-gnu/libnftnl.so.11* \
|
||||
/lib/x86_64-linux-gnu/libc.so.6 \
|
||||
/lib64/ld-linux-x86-64.so.2; do
|
||||
[ -e "$lib" ] && cp -aL "$lib" "$ROOTFS${lib}" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Copy xtables modules directory (match extensions)
|
||||
if [ -d /usr/lib/x86_64-linux-gnu/xtables ]; then
|
||||
mkdir -p "$ROOTFS/usr/lib/x86_64-linux-gnu/xtables"
|
||||
cp -a /usr/lib/x86_64-linux-gnu/xtables/*.so "$ROOTFS/usr/lib/x86_64-linux-gnu/xtables/" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo " Installed iptables-nft (xtables-nft-multi) + shared libs"
|
||||
else
|
||||
echo " WARN: xtables-nft-multi not found in builder (install iptables package)"
|
||||
fi
|
||||
|
||||
# Kernel modules list (for init to load at boot)
|
||||
cp "$PROJECT_ROOT/build/config/modules.list" "$ROOTFS/usr/lib/kubesolo-os/modules.list"
|
||||
|
||||
# --- 4. Sysctl config ---
|
||||
@@ -139,6 +369,6 @@ fi
|
||||
echo ""
|
||||
echo "==> Injection complete. Rootfs contents:"
|
||||
echo " Total size: $(du -sh "$ROOTFS" | cut -f1)"
|
||||
echo " KubeSolo: $(du -h "$ROOTFS/usr/local/bin/kubesolo" | cut -f1)"
|
||||
echo " KubeSolo: $(du -h "$ROOTFS/usr/bin/kubesolo" | cut -f1)"
|
||||
echo " Init stages: $(ls "$ROOTFS/usr/lib/kubesolo-os/init.d/" | wc -l)"
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user