feat: custom kernel build + boot fixes for working container runtime
Build a custom Tiny Core 17.0 kernel (6.18.2) with missing configs that the stock kernel lacks for container workloads: - CONFIG_CGROUP_BPF=y (cgroup v2 device control via BPF) - CONFIG_DEVTMPFS=y (auto-create /dev device nodes) - CONFIG_DEVTMPFS_MOUNT=y (auto-mount devtmpfs) - CONFIG_MEMCG=y (memory cgroup controller for memory.max) - CONFIG_CFS_BANDWIDTH=y (CPU bandwidth throttling for cpu.max) Also strips unnecessary subsystems (sound, GPU, wireless, Bluetooth, KVM, etc.) for minimal footprint on a headless K8s edge appliance. Init system fixes for successful boot-to-running-pods: - Add switch_root in init.sh to escape initramfs (runc pivot_root) - Add mountpoint guards in 00-early-mount.sh (skip if already mounted) - Create essential device nodes after switch_root (kmsg, console, etc.) - Enable cgroup v2 controller delegation with init process isolation - Mount BPF filesystem for cgroup v2 device control - Add mknod fallback from sysfs in 20-persistent-mount.sh for /dev/vda - Move KubeSolo binary to /usr/bin (avoid /usr/local bind mount hiding) - Generate /etc/machine-id in 60-hostname.sh (kubelet requires it) - Pre-initialize iptables tables before kube-proxy starts - Add nft_reject, nft_fib, xt_nfacct to kernel modules list Build system changes: - New build-kernel.sh script for custom kernel compilation - Dockerfile.builder adds kernel build deps (flex, bison, libelf, etc.) - Selective kernel module install (only modules.list + transitive deps) - Install iptables-nft (xtables-nft-multi) + shared libs in rootfs Tested: ISO boots in QEMU, node reaches Ready in ~35s, CoreDNS and local-path-provisioner pods start and run successfully. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -31,36 +31,126 @@ else
|
||||
fi
|
||||
|
||||
# --- KubeSolo ---
|
||||
KUBESOLO_INSTALLER="$CACHE_DIR/install-kubesolo.sh"
|
||||
KUBESOLO_VERSION="${KUBESOLO_VERSION:-v1.1.0}"
|
||||
KUBESOLO_BIN="$CACHE_DIR/kubesolo"
|
||||
|
||||
if [ -f "$KUBESOLO_BIN" ]; then
|
||||
echo "==> KubeSolo binary already cached: $KUBESOLO_BIN"
|
||||
else
|
||||
echo "==> Downloading KubeSolo installer..."
|
||||
curl -sfL "$KUBESOLO_INSTALL_URL" -o "$KUBESOLO_INSTALLER"
|
||||
echo "==> Downloading KubeSolo ${KUBESOLO_VERSION}..."
|
||||
|
||||
echo "==> Extracting KubeSolo binary..."
|
||||
echo " NOTE: The installer normally runs 'install'. We extract the binary URL instead."
|
||||
echo " For Phase 1 PoC, install KubeSolo on a host and copy the binary."
|
||||
echo ""
|
||||
echo " Manual step required:"
|
||||
echo " 1. On a Linux x86_64 host: curl -sfL https://get.kubesolo.io | sudo sh -"
|
||||
echo " 2. Copy /usr/local/bin/kubesolo to: $KUBESOLO_BIN"
|
||||
echo " 3. Re-run: make rootfs"
|
||||
echo ""
|
||||
# Determine architecture
|
||||
ARCH="${TARGET_ARCH:-amd64}"
|
||||
OS="linux"
|
||||
|
||||
# Try to extract download URL from installer script
|
||||
BINARY_URL=$(grep -oP 'https://[^ ]+kubesolo[^ ]+' "$KUBESOLO_INSTALLER" 2>/dev/null | head -1 || true)
|
||||
if [ -n "$BINARY_URL" ]; then
|
||||
echo " Attempting direct download from: $BINARY_URL"
|
||||
curl -sfL "$BINARY_URL" -o "$KUBESOLO_BIN" && chmod +x "$KUBESOLO_BIN" || {
|
||||
echo " Direct download failed. Use manual step above."
|
||||
}
|
||||
# Build download URL from GitHub releases
|
||||
# Available variants: kubesolo-v1.1.0-linux-amd64.tar.gz, kubesolo-v1.1.0-linux-amd64-musl.tar.gz
|
||||
# We use the musl variant for maximum compatibility with Tiny Core Linux (musl-based)
|
||||
BIN_URL="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-${OS}-${ARCH}-musl.tar.gz"
|
||||
BIN_URL_FALLBACK="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-${OS}-${ARCH}.tar.gz"
|
||||
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf '$TEMP_DIR'" EXIT
|
||||
|
||||
echo " URL: $BIN_URL"
|
||||
if curl -fSL "$BIN_URL" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||
echo " Downloaded musl variant"
|
||||
elif curl -fSL "$BIN_URL_FALLBACK" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||
echo " Downloaded glibc variant (fallback)"
|
||||
else
|
||||
echo "ERROR: Failed to download KubeSolo from GitHub."
|
||||
echo " Tried: $BIN_URL"
|
||||
echo " Tried: $BIN_URL_FALLBACK"
|
||||
echo ""
|
||||
echo " Manual step:"
|
||||
echo " 1. Download from: https://github.com/portainer/kubesolo/releases"
|
||||
echo " 2. Extract and copy binary to: $KUBESOLO_BIN"
|
||||
echo " 3. Re-run: make rootfs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -f "$KUBESOLO_BIN" ]; then
|
||||
echo "==> KubeSolo binary: $KUBESOLO_BIN ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
||||
# Extract binary from tarball
|
||||
tar -xzf "$TEMP_DIR/kubesolo.tar.gz" -C "$TEMP_DIR"
|
||||
|
||||
# Find the kubesolo binary in extracted contents
|
||||
FOUND_BIN=$(find "$TEMP_DIR" -name "kubesolo" -type f ! -name "*.tar.gz" | head -1)
|
||||
if [ -z "$FOUND_BIN" ]; then
|
||||
echo "ERROR: Could not find kubesolo binary in extracted archive"
|
||||
echo " Archive contents:"
|
||||
ls -la "$TEMP_DIR"/
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cp "$FOUND_BIN" "$KUBESOLO_BIN"
|
||||
chmod +x "$KUBESOLO_BIN"
|
||||
|
||||
trap - EXIT
|
||||
rm -rf "$TEMP_DIR"
|
||||
|
||||
echo "==> KubeSolo binary: $KUBESOLO_BIN ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
||||
fi
|
||||
|
||||
# --- Tiny Core kernel module extensions (netfilter, iptables) ---
|
||||
# The base Tiny Core initramfs does NOT include netfilter kernel modules.
|
||||
# They are distributed as separate TCZ (squashfs) extensions.
|
||||
# KubeSolo requires netfilter for kube-proxy, iptables NAT, conntrack, etc.
|
||||
|
||||
# Detect kernel version from the cached ISO
|
||||
KVER=""
|
||||
if [ -f "$TC_ISO" ]; then
|
||||
# Try to detect kernel version from ISO without mounting
|
||||
# Tiny Core 17.0 uses 6.18.2-tinycore64
|
||||
KVER="6.18.2-tinycore64"
|
||||
fi
|
||||
|
||||
NETFILTER_TCZ="$CACHE_DIR/ipv6-netfilter-${KVER}.tcz"
|
||||
NETFILTER_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/ipv6-netfilter-${KVER}.tcz"
|
||||
|
||||
if [ -f "$NETFILTER_TCZ" ]; then
|
||||
echo "==> Netfilter modules already cached: $NETFILTER_TCZ"
|
||||
else
|
||||
echo "==> Downloading netfilter kernel modules (ipv6-netfilter-${KVER}.tcz)..."
|
||||
echo " URL: $NETFILTER_TCZ_URL"
|
||||
if wget -q --show-progress -O "$NETFILTER_TCZ" "$NETFILTER_TCZ_URL" 2>/dev/null || \
|
||||
curl -fSL "$NETFILTER_TCZ_URL" -o "$NETFILTER_TCZ" 2>/dev/null; then
|
||||
echo "==> Downloaded: $NETFILTER_TCZ ($(du -h "$NETFILTER_TCZ" | cut -f1))"
|
||||
else
|
||||
echo "WARN: Failed to download netfilter modules. kube-proxy may not work."
|
||||
rm -f "$NETFILTER_TCZ"
|
||||
fi
|
||||
fi
|
||||
|
||||
NET_BRIDGING_TCZ="$CACHE_DIR/net-bridging-${KVER}.tcz"
|
||||
NET_BRIDGING_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/net-bridging-${KVER}.tcz"
|
||||
|
||||
if [ -f "$NET_BRIDGING_TCZ" ]; then
|
||||
echo "==> Net-bridging modules already cached: $NET_BRIDGING_TCZ"
|
||||
else
|
||||
echo "==> Downloading net-bridging kernel modules (net-bridging-${KVER}.tcz)..."
|
||||
echo " URL: $NET_BRIDGING_TCZ_URL"
|
||||
if wget -q --show-progress -O "$NET_BRIDGING_TCZ" "$NET_BRIDGING_TCZ_URL" 2>/dev/null || \
|
||||
curl -fSL "$NET_BRIDGING_TCZ_URL" -o "$NET_BRIDGING_TCZ" 2>/dev/null; then
|
||||
echo "==> Downloaded: $NET_BRIDGING_TCZ ($(du -h "$NET_BRIDGING_TCZ" | cut -f1))"
|
||||
else
|
||||
echo "WARN: Failed to download net-bridging modules. CNI bridge may not work."
|
||||
rm -f "$NET_BRIDGING_TCZ"
|
||||
fi
|
||||
fi
|
||||
|
||||
IPTABLES_TCZ="$CACHE_DIR/iptables.tcz"
|
||||
IPTABLES_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/iptables.tcz"
|
||||
|
||||
if [ -f "$IPTABLES_TCZ" ]; then
|
||||
echo "==> iptables userspace already cached: $IPTABLES_TCZ"
|
||||
else
|
||||
echo "==> Downloading iptables userspace tools..."
|
||||
echo " URL: $IPTABLES_TCZ_URL"
|
||||
if wget -q --show-progress -O "$IPTABLES_TCZ" "$IPTABLES_TCZ_URL" 2>/dev/null || \
|
||||
curl -fSL "$IPTABLES_TCZ_URL" -o "$IPTABLES_TCZ" 2>/dev/null; then
|
||||
echo "==> Downloaded: $IPTABLES_TCZ ($(du -h "$IPTABLES_TCZ" | cut -f1))"
|
||||
else
|
||||
echo "WARN: Failed to download iptables. KubeSolo bundles its own but this is a fallback."
|
||||
rm -f "$IPTABLES_TCZ"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user