feat: custom kernel build + boot fixes for working container runtime
Build a custom Tiny Core 17.0 kernel (6.18.2) with missing configs that the stock kernel lacks for container workloads: - CONFIG_CGROUP_BPF=y (cgroup v2 device control via BPF) - CONFIG_DEVTMPFS=y (auto-create /dev device nodes) - CONFIG_DEVTMPFS_MOUNT=y (auto-mount devtmpfs) - CONFIG_MEMCG=y (memory cgroup controller for memory.max) - CONFIG_CFS_BANDWIDTH=y (CPU bandwidth throttling for cpu.max) Also strips unnecessary subsystems (sound, GPU, wireless, Bluetooth, KVM, etc.) for minimal footprint on a headless K8s edge appliance. Init system fixes for successful boot-to-running-pods: - Add switch_root in init.sh to escape initramfs (runc pivot_root) - Add mountpoint guards in 00-early-mount.sh (skip if already mounted) - Create essential device nodes after switch_root (kmsg, console, etc.) - Enable cgroup v2 controller delegation with init process isolation - Mount BPF filesystem for cgroup v2 device control - Add mknod fallback from sysfs in 20-persistent-mount.sh for /dev/vda - Move KubeSolo binary to /usr/bin (avoid /usr/local bind mount hiding) - Generate /etc/machine-id in 60-hostname.sh (kubelet requires it) - Pre-initialize iptables tables before kube-proxy starts - Add nft_reject, nft_fib, xt_nfacct to kernel modules list Build system changes: - New build-kernel.sh script for custom kernel compilation - Dockerfile.builder adds kernel build deps (flex, bison, libelf, etc.) - Selective kernel module install (only modules.list + transitive deps) - Install iptables-nft (xtables-nft-multi) + shared libs in rootfs Tested: ISO boots in QEMU, node reaches Ready in ~35s, CoreDNS and local-path-provisioner pods start and run successfully. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
209
build/scripts/build-kernel.sh
Executable file
209
build/scripts/build-kernel.sh
Executable file
@@ -0,0 +1,209 @@
|
||||
#!/bin/bash
|
||||
# build-kernel.sh — Build custom Tiny Core kernel with CONFIG_CGROUP_BPF=y
|
||||
#
|
||||
# The stock Tiny Core 17.0 kernel (6.18.2-tinycore64) lacks CONFIG_CGROUP_BPF,
|
||||
# which is required for cgroup v2 device control in runc/containerd.
|
||||
# This script downloads the TC-patched kernel source, enables CONFIG_CGROUP_BPF,
|
||||
# and builds vmlinuz + modules.
|
||||
#
|
||||
# Output is cached in $CACHE_DIR/custom-kernel/ and reused across builds.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
||||
|
||||
# shellcheck source=../config/versions.env
|
||||
. "$SCRIPT_DIR/../config/versions.env"
|
||||
|
||||
KVER="6.18.2-tinycore64"
|
||||
KERNEL_BASE_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/release/src/kernel"
|
||||
KERNEL_SRC_URL="${KERNEL_BASE_URL}/linux-6.18.2-patched.tar.xz"
|
||||
KERNEL_CFG_URL="${KERNEL_BASE_URL}/config-${KVER}"
|
||||
|
||||
CUSTOM_KERNEL_DIR="$CACHE_DIR/custom-kernel"
|
||||
CUSTOM_VMLINUZ="$CUSTOM_KERNEL_DIR/vmlinuz"
|
||||
CUSTOM_MODULES="$CUSTOM_KERNEL_DIR/modules"
|
||||
|
||||
mkdir -p "$CACHE_DIR" "$CUSTOM_KERNEL_DIR"
|
||||
|
||||
# --- Skip if already built ---
|
||||
if [ -f "$CUSTOM_VMLINUZ" ] && [ -d "$CUSTOM_MODULES/lib/modules/$KVER" ]; then
|
||||
echo "==> Custom kernel already built (cached)"
|
||||
echo " vmlinuz: $CUSTOM_VMLINUZ ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||
MOD_COUNT=$(find "$CUSTOM_MODULES/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||
echo " Modules: $MOD_COUNT modules in $CUSTOM_MODULES/lib/modules/$KVER"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "==> Building custom kernel with CONFIG_CGROUP_BPF=y..."
|
||||
echo " Kernel version: $KVER"
|
||||
|
||||
# --- Download kernel source ---
|
||||
KERNEL_SRC_ARCHIVE="$CACHE_DIR/linux-6.18.2-patched.tar.xz"
|
||||
if [ ! -f "$KERNEL_SRC_ARCHIVE" ]; then
|
||||
echo "==> Downloading kernel source (~149 MB)..."
|
||||
echo " URL: $KERNEL_SRC_URL"
|
||||
wget -q --show-progress -O "$KERNEL_SRC_ARCHIVE" "$KERNEL_SRC_URL" 2>/dev/null || \
|
||||
curl -fSL "$KERNEL_SRC_URL" -o "$KERNEL_SRC_ARCHIVE"
|
||||
echo " Downloaded: $(du -h "$KERNEL_SRC_ARCHIVE" | cut -f1)"
|
||||
else
|
||||
echo "==> Kernel source already cached: $(du -h "$KERNEL_SRC_ARCHIVE" | cut -f1)"
|
||||
fi
|
||||
|
||||
# --- Download stock config ---
|
||||
KERNEL_CFG="$CACHE_DIR/config-${KVER}"
|
||||
if [ ! -f "$KERNEL_CFG" ]; then
|
||||
echo "==> Downloading stock kernel config..."
|
||||
echo " URL: $KERNEL_CFG_URL"
|
||||
wget -q -O "$KERNEL_CFG" "$KERNEL_CFG_URL" 2>/dev/null || \
|
||||
curl -fSL "$KERNEL_CFG_URL" -o "$KERNEL_CFG"
|
||||
else
|
||||
echo "==> Stock kernel config already cached"
|
||||
fi
|
||||
|
||||
# --- Extract source ---
|
||||
# IMPORTANT: Must extract on a case-sensitive filesystem. The kernel source has
|
||||
# files that differ only by case (e.g., xt_mark.h vs xt_MARK.h). If the cache
|
||||
# is on macOS (case-insensitive APFS), extraction silently loses files.
|
||||
# Use /tmp inside the container (ext4, case-sensitive) for the build.
|
||||
KERNEL_BUILD_DIR="/tmp/kernel-build"
|
||||
rm -rf "$KERNEL_BUILD_DIR"
|
||||
mkdir -p "$KERNEL_BUILD_DIR"
|
||||
|
||||
echo "==> Extracting kernel source (case-sensitive filesystem)..."
|
||||
tar -xf "$KERNEL_SRC_ARCHIVE" -C "$KERNEL_BUILD_DIR"
|
||||
|
||||
# Find the extracted source directory (could be linux-6.18.2 or linux-6.18.2-patched)
|
||||
KERNEL_SRC_DIR=$(find "$KERNEL_BUILD_DIR" -maxdepth 1 -type d -name 'linux-*' | head -1)
|
||||
if [ -z "$KERNEL_SRC_DIR" ]; then
|
||||
echo "ERROR: Could not find kernel source directory after extraction"
|
||||
ls -la "$KERNEL_BUILD_DIR"/
|
||||
exit 1
|
||||
fi
|
||||
echo " Source dir: $(basename "$KERNEL_SRC_DIR")"
|
||||
|
||||
cd "$KERNEL_SRC_DIR"
|
||||
|
||||
# --- Apply stock config + enable CONFIG_CGROUP_BPF ---
|
||||
echo "==> Applying stock Tiny Core config..."
|
||||
cp "$KERNEL_CFG" .config
|
||||
|
||||
echo "==> Enabling required kernel configs..."
|
||||
./scripts/config --enable CONFIG_CGROUP_BPF
|
||||
./scripts/config --enable CONFIG_DEVTMPFS
|
||||
./scripts/config --enable CONFIG_DEVTMPFS_MOUNT
|
||||
./scripts/config --enable CONFIG_MEMCG
|
||||
./scripts/config --enable CONFIG_CFS_BANDWIDTH
|
||||
|
||||
# --- Strip unnecessary subsystems for smallest footprint ---
|
||||
# This is a headless K8s edge appliance — no sound, GPU, wireless, etc.
|
||||
echo "==> Disabling unnecessary subsystems for minimal footprint..."
|
||||
|
||||
# Sound subsystem (not needed on headless appliance)
|
||||
./scripts/config --disable SOUND
|
||||
|
||||
# GPU/DRM (serial console only, no display)
|
||||
./scripts/config --disable DRM
|
||||
|
||||
# KVM hypervisor (this IS the guest/bare metal, not a hypervisor)
|
||||
./scripts/config --disable KVM
|
||||
|
||||
# Media/camera/TV/radio (not needed)
|
||||
./scripts/config --disable MEDIA_SUPPORT
|
||||
|
||||
# Wireless networking (wired edge device)
|
||||
./scripts/config --disable WIRELESS
|
||||
./scripts/config --disable WLAN
|
||||
./scripts/config --disable CFG80211
|
||||
|
||||
# Bluetooth (not needed)
|
||||
./scripts/config --disable BT
|
||||
|
||||
# NFC (not needed)
|
||||
./scripts/config --disable NFC
|
||||
|
||||
# Infiniband (not needed on edge)
|
||||
./scripts/config --disable INFINIBAND
|
||||
|
||||
# PCMCIA (legacy, not needed)
|
||||
./scripts/config --disable PCMCIA
|
||||
|
||||
# Amateur radio (not needed)
|
||||
./scripts/config --disable HAMRADIO
|
||||
|
||||
# ISDN (not needed)
|
||||
./scripts/config --disable ISDN
|
||||
|
||||
# ATM networking (not needed)
|
||||
./scripts/config --disable ATM
|
||||
|
||||
# Joystick/gamepad (not needed)
|
||||
./scripts/config --disable INPUT_JOYSTICK
|
||||
./scripts/config --disable INPUT_TABLET
|
||||
|
||||
# FPGA (not needed)
|
||||
./scripts/config --disable FPGA
|
||||
|
||||
# Resolve dependencies (olddefconfig accepts defaults for new options)
|
||||
make olddefconfig
|
||||
|
||||
# Verify CONFIG_CGROUP_BPF is set
|
||||
if grep -q 'CONFIG_CGROUP_BPF=y' .config; then
|
||||
echo " CONFIG_CGROUP_BPF=y confirmed in .config"
|
||||
else
|
||||
echo "ERROR: CONFIG_CGROUP_BPF not set after olddefconfig"
|
||||
grep 'CGROUP_BPF' .config || echo " (CGROUP_BPF not found in .config)"
|
||||
echo ""
|
||||
echo "Prerequisites check:"
|
||||
grep -E 'CONFIG_BPF=|CONFIG_BPF_SYSCALL=' .config || echo " BPF not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Show what changed
|
||||
echo " Config diff from stock:"
|
||||
diff "$KERNEL_CFG" .config | grep '^[<>]' | head -20 || echo " (no differences beyond CGROUP_BPF)"
|
||||
|
||||
# --- Build kernel + modules ---
|
||||
NPROC=$(nproc 2>/dev/null || echo 4)
|
||||
echo ""
|
||||
echo "==> Building kernel (${NPROC} parallel jobs)..."
|
||||
echo " This may take 15-25 minutes..."
|
||||
|
||||
make -j"$NPROC" bzImage modules 2>&1
|
||||
|
||||
echo "==> Kernel build complete"
|
||||
|
||||
# --- Install to staging ---
|
||||
echo "==> Installing vmlinuz..."
|
||||
cp arch/x86/boot/bzImage "$CUSTOM_VMLINUZ"
|
||||
|
||||
echo "==> Installing modules (stripped)..."
|
||||
rm -rf "$CUSTOM_MODULES"
|
||||
mkdir -p "$CUSTOM_MODULES"
|
||||
make INSTALL_MOD_STRIP=1 modules_install INSTALL_MOD_PATH="$CUSTOM_MODULES"
|
||||
|
||||
# Remove build/source symlinks (they point to the build dir which won't exist in rootfs)
|
||||
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/build"
|
||||
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/source"
|
||||
|
||||
# Run depmod to generate proper module dependency files
|
||||
echo "==> Running depmod..."
|
||||
depmod -a -b "$CUSTOM_MODULES" "$KVER" 2>/dev/null || true
|
||||
|
||||
# Save the final config for reference
|
||||
cp .config "$CUSTOM_KERNEL_DIR/.config"
|
||||
|
||||
# --- Clean up build dir (large, ~1.5 GB) ---
|
||||
echo "==> Cleaning kernel build directory..."
|
||||
cd /
|
||||
rm -rf "$KERNEL_BUILD_DIR"
|
||||
|
||||
# --- Summary ---
|
||||
echo ""
|
||||
echo "==> Custom kernel build complete:"
|
||||
echo " vmlinuz: $CUSTOM_VMLINUZ ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||
MOD_COUNT=$(find "$CUSTOM_MODULES/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||
echo " Modules: $MOD_COUNT modules"
|
||||
echo " Modules size: $(du -sh "$CUSTOM_MODULES/lib/modules/$KVER" | cut -f1)"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user