feat: initial Phase 1 PoC scaffolding for KubeSolo OS

Complete Phase 1 implementation of KubeSolo OS — an immutable, bootable Linux distribution built on Tiny Core Linux for running KubeSolo single-node Kubernetes. Build system: - Makefile with fetch, rootfs, initramfs, iso, disk-image targets - Dockerfile.builder for reproducible builds - Scripts to download Tiny Core, extract rootfs, inject KubeSolo, pack initramfs, and create bootable ISO/disk images Init system (10 POSIX sh stages): - Early mount (proc/sys/dev/cgroup2), cmdline parsing, persistent mount with bind-mounts, kernel module loading, sysctl, DHCP networking, hostname, clock sync, containerd prep, KubeSolo exec Shared libraries: - functions.sh (device wait, IP lookup, config helpers) - network.sh (static IP, config persistence, interface detection) - health.sh (containerd, API server, node readiness checks) - Emergency shell for boot failure debugging Testing: - QEMU boot test with serial log marker detection - K8s readiness test with kubectl verification - Persistence test (reboot + verify state survives) - Workload deployment test (nginx pod) - Local storage test (PVC + local-path provisioner) - Network policy test - Reusable run-vm.sh launcher Developer tools: - dev-vm.sh (interactive QEMU with port forwarding) - rebuild-initramfs.sh (fast iteration) - inject-ssh.sh (dropbear SSH for debugging) - extract-kernel-config.sh + kernel-audit.sh Documentation: - Full design document with architecture research - Boot flow documentation covering all 10 init stages - Cloud-init examples (DHCP, static IP, Portainer Edge, air-gapped) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-11 10:18:42 -06:00
commit e372df578b
50 changed files with 4392 additions and 0 deletions
--- a/init/emergency-shell.sh
+++ b/init/emergency-shell.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+# emergency-shell.sh — Drop to a debug shell on boot failure
+# Called by init when a critical stage fails
+# POSIX sh only — BusyBox ash compatible
+
+echo "" >&2
+echo "=====================================================" >&2
+echo "  KubeSolo OS — Emergency Shell" >&2
+echo "=====================================================" >&2
+echo "" >&2
+echo "  The boot process has failed. You have been dropped" >&2
+echo "  into an emergency shell for debugging." >&2
+echo "" >&2
+echo "  Useful commands:" >&2
+echo "    dmesg | tail -50       Kernel messages" >&2
+echo "    cat /proc/cmdline      Boot parameters" >&2
+echo "    cat /proc/mounts       Current mounts" >&2
+echo "    blkid                  Block device info" >&2
+echo "    ip addr                Network interfaces" >&2
+echo "    ls /usr/lib/kubesolo-os/init.d/  Init stages" >&2
+echo "" >&2
+
+# Show version if available
+if [ -f /etc/kubesolo-os-version ]; then
+    echo "  OS Version: $(cat /etc/kubesolo-os-version)" >&2
+fi
+
+# Show what stage failed if known
+if [ -n "${FAILED_STAGE:-}" ]; then
+    echo "  Failed stage: $FAILED_STAGE" >&2
+fi
+
+echo "" >&2
+echo "  Type 'exit' to attempt re-running the init sequence." >&2
+echo "  Type 'reboot' to restart the system." >&2
+echo "=====================================================" >&2
+echo "" >&2
+
+# Ensure basic env is usable
+export PATH="/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin"
+export PS1="[kubesolo-emergency] # "
+export HOME=/root
+export TERM="${TERM:-linux}"
+
+# Create home dir if needed
+mkdir -p /root
+
+exec /bin/sh
--- a/init/init.sh
+++ b/init/init.sh
@@ -0,0 +1,87 @@
+#!/bin/sh
+# /sbin/init — KubeSolo OS init system
+# POSIX sh compatible (BusyBox ash)
+#
+# Boot stages are sourced from /usr/lib/kubesolo-os/init.d/ in numeric order.
+# Each stage file must be a valid POSIX sh script.
+# If any mandatory stage fails, the system drops to an emergency shell.
+#
+# Boot parameters (from kernel command line):
+#   kubesolo.data=<device>    Persistent data partition (required)
+#   kubesolo.debug            Enable verbose logging
+#   kubesolo.shell            Drop to emergency shell immediately
+#   kubesolo.nopersist        Run without persistent storage (RAM only)
+#   kubesolo.cloudinit=<path> Path to cloud-init config
+#   kubesolo.flags=<flags>    Extra flags for KubeSolo binary
+
+set -e
+
+# --- Constants ---
+INIT_LIB="/usr/lib/kubesolo-os"
+INIT_STAGES="/usr/lib/kubesolo-os/init.d"
+LOG_PREFIX="[kubesolo-init]"
+DATA_MOUNT="/mnt/data"
+
+# --- Parsed boot parameters (populated by 10-parse-cmdline.sh) ---
+export KUBESOLO_DATA_DEV=""
+export KUBESOLO_DEBUG=""
+export KUBESOLO_SHELL=""
+export KUBESOLO_NOPERSIST=""
+export KUBESOLO_CLOUDINIT=""
+export KUBESOLO_EXTRA_FLAGS=""
+
+# --- Logging ---
+log() {
+    echo "$LOG_PREFIX $*" >&2
+}
+
+log_ok() {
+    echo "$LOG_PREFIX [OK] $*" >&2
+}
+
+log_err() {
+    echo "$LOG_PREFIX [ERROR] $*" >&2
+}
+
+log_warn() {
+    echo "$LOG_PREFIX [WARN] $*" >&2
+}
+
+# --- Emergency shell ---
+emergency_shell() {
+    log_err "Boot failed: $*"
+    log_err "Dropping to emergency shell. Type 'exit' to retry boot."
+    exec /bin/sh
+}
+
+# --- Main boot sequence ---
+log "KubeSolo OS v$(cat /etc/kubesolo-os-version 2>/dev/null || echo 'dev') starting..."
+
+# Source shared functions
+if [ -f "$INIT_LIB/functions.sh" ]; then
+    . "$INIT_LIB/functions.sh"
+fi
+
+# Run init stages in order
+for stage in "$INIT_STAGES"/*.sh; do
+    [ -f "$stage" ] || continue
+    stage_name="$(basename "$stage")"
+
+    log "Running stage: $stage_name"
+
+    if ! . "$stage"; then
+        emergency_shell "Stage $stage_name failed"
+    fi
+
+    # Check for early shell request (parsed in 10-parse-cmdline.sh)
+    if [ "$KUBESOLO_SHELL" = "1" ] && [ "$stage_name" = "10-parse-cmdline.sh" ]; then
+        log "Emergency shell requested via boot parameter"
+        exec /bin/sh
+    fi
+
+    log_ok "Stage $stage_name complete"
+done
+
+# If we get here, all stages ran but KubeSolo should have exec'd.
+# This means 90-kubesolo.sh didn't exec (shouldn't happen).
+emergency_shell "Init completed without exec'ing KubeSolo — this is a bug"
--- a/init/lib/00-early-mount.sh
+++ b/init/lib/00-early-mount.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+# 00-early-mount.sh — Mount essential virtual filesystems
+
+mount -t proc proc /proc 2>/dev/null || true
+mount -t sysfs sysfs /sys 2>/dev/null || true
+mount -t devtmpfs devtmpfs /dev 2>/dev/null || true
+mount -t tmpfs tmpfs /tmp
+mount -t tmpfs tmpfs /run
+
+mkdir -p /dev/pts /dev/shm
+mount -t devpts devpts /dev/pts
+mount -t tmpfs tmpfs /dev/shm
+
+# Mount cgroup2 unified hierarchy
+mkdir -p /sys/fs/cgroup
+mount -t cgroup2 cgroup2 /sys/fs/cgroup 2>/dev/null || {
+    log_warn "cgroup v2 mount failed; attempting v1 fallback"
+    mount -t tmpfs cgroup /sys/fs/cgroup
+    for subsys in cpu cpuacct memory devices freezer pids; do
+        mkdir -p "/sys/fs/cgroup/$subsys"
+        mount -t cgroup -o "$subsys" "cgroup_${subsys}" "/sys/fs/cgroup/$subsys" 2>/dev/null || true
+    done
+}
--- a/init/lib/10-parse-cmdline.sh
+++ b/init/lib/10-parse-cmdline.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+# 10-parse-cmdline.sh — Parse boot parameters from /proc/cmdline
+
+for arg in $(cat /proc/cmdline); do
+    case "$arg" in
+        kubesolo.data=*)      KUBESOLO_DATA_DEV="${arg#kubesolo.data=}" ;;
+        kubesolo.debug)       KUBESOLO_DEBUG=1; set -x ;;
+        kubesolo.shell)       KUBESOLO_SHELL=1 ;;
+        kubesolo.nopersist)   KUBESOLO_NOPERSIST=1 ;;
+        kubesolo.cloudinit=*) KUBESOLO_CLOUDINIT="${arg#kubesolo.cloudinit=}" ;;
+        kubesolo.flags=*)     KUBESOLO_EXTRA_FLAGS="${arg#kubesolo.flags=}" ;;
+    esac
+done
+
+if [ -z "$KUBESOLO_DATA_DEV" ] && [ "$KUBESOLO_NOPERSIST" != "1" ]; then
+    log_warn "No kubesolo.data= specified and kubesolo.nopersist not set"
+    log_warn "Attempting auto-detection of data partition (label: KSOLODATA)"
+    KUBESOLO_DATA_DEV=$(blkid -L KSOLODATA 2>/dev/null || true)
+    if [ -z "$KUBESOLO_DATA_DEV" ]; then
+        log_warn "No data partition found. Running in RAM-only mode."
+        KUBESOLO_NOPERSIST=1
+    else
+        log "Auto-detected data partition: $KUBESOLO_DATA_DEV"
+    fi
+fi
+
+log "Config: data=$KUBESOLO_DATA_DEV debug=$KUBESOLO_DEBUG nopersist=$KUBESOLO_NOPERSIST"
--- a/init/lib/20-persistent-mount.sh
+++ b/init/lib/20-persistent-mount.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+# 20-persistent-mount.sh — Mount persistent data partition and bind-mount writable paths
+
+if [ "$KUBESOLO_NOPERSIST" = "1" ]; then
+    log "Running in RAM-only mode — no persistent storage"
+    # Create tmpfs-backed directories so KubeSolo has somewhere to write
+    mkdir -p /var/lib/kubesolo /var/lib/containerd /etc/kubesolo /var/log /usr/local
+    return 0
+fi
+
+# Wait for device to appear (USB, slow disks, virtio)
+log "Waiting for data device: $KUBESOLO_DATA_DEV"
+WAIT_SECS=30
+for i in $(seq 1 "$WAIT_SECS"); do
+    [ -b "$KUBESOLO_DATA_DEV" ] && break
+    sleep 1
+done
+
+if [ ! -b "$KUBESOLO_DATA_DEV" ]; then
+    log_err "Data device $KUBESOLO_DATA_DEV not found after ${WAIT_SECS}s"
+    return 1
+fi
+
+# Mount data partition
+mkdir -p "$DATA_MOUNT"
+mount -t ext4 -o noatime "$KUBESOLO_DATA_DEV" "$DATA_MOUNT" || {
+    log_err "Failed to mount $KUBESOLO_DATA_DEV"
+    return 1
+}
+log_ok "Mounted $KUBESOLO_DATA_DEV at $DATA_MOUNT"
+
+# Create persistent directory structure (first boot)
+for dir in kubesolo containerd etc-kubesolo log usr-local network; do
+    mkdir -p "$DATA_MOUNT/$dir"
+done
+
+# Ensure target mount points exist
+mkdir -p /var/lib/kubesolo /var/lib/containerd /etc/kubesolo /var/log /usr/local
+
+# Bind mount persistent paths
+mount --bind "$DATA_MOUNT/kubesolo"     /var/lib/kubesolo
+mount --bind "$DATA_MOUNT/containerd"   /var/lib/containerd
+mount --bind "$DATA_MOUNT/etc-kubesolo" /etc/kubesolo
+mount --bind "$DATA_MOUNT/log"          /var/log
+mount --bind "$DATA_MOUNT/usr-local"    /usr/local
+
+log_ok "Persistent bind mounts configured"
--- a/init/lib/30-kernel-modules.sh
+++ b/init/lib/30-kernel-modules.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+# 30-kernel-modules.sh — Load required kernel modules for K8s
+
+MODULES_LIST="/usr/lib/kubesolo-os/modules.list"
+
+if [ ! -f "$MODULES_LIST" ]; then
+    log_warn "No modules list found at $MODULES_LIST"
+    return 0
+fi
+
+LOADED=0
+FAILED=0
+
+while IFS= read -r mod; do
+    # Skip comments and blank lines
+    case "$mod" in
+        '#'*|'') continue ;;
+    esac
+    mod="$(echo "$mod" | tr -d '[:space:]')"
+    if modprobe "$mod" 2>/dev/null; then
+        LOADED=$((LOADED + 1))
+    else
+        log_warn "Failed to load module: $mod (may be built-in)"
+        FAILED=$((FAILED + 1))
+    fi
+done < "$MODULES_LIST"
+
+log_ok "Loaded $LOADED modules ($FAILED failed/built-in)"
--- a/init/lib/40-sysctl.sh
+++ b/init/lib/40-sysctl.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+# 40-sysctl.sh — Apply kernel parameters required for K8s networking
+
+# Apply all .conf files in sysctl.d
+for conf in /etc/sysctl.d/*.conf; do
+    [ -f "$conf" ] || continue
+    while IFS='=' read -r key value; do
+        case "$key" in
+            '#'*|'') continue ;;
+        esac
+        key="$(echo "$key" | tr -d '[:space:]')"
+        value="$(echo "$value" | tr -d '[:space:]')"
+        if [ -n "$key" ] && [ -n "$value" ]; then
+            sysctl -w "${key}=${value}" >/dev/null 2>&1 || \
+                log_warn "Failed to set sysctl: ${key}=${value}"
+        fi
+    done < "$conf"
+done
+
+log_ok "Sysctl settings applied"
--- a/init/lib/50-network.sh
+++ b/init/lib/50-network.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+# 50-network.sh — Configure networking
+# Priority: persistent config > cloud-init > DHCP fallback
+
+# Check for saved network config (from previous boot or cloud-init)
+if [ -f "$DATA_MOUNT/network/interfaces.sh" ]; then
+    log "Applying saved network configuration"
+    . "$DATA_MOUNT/network/interfaces.sh"
+    return 0
+fi
+
+# Check for cloud-init network config
+CLOUDINIT_FILE="${KUBESOLO_CLOUDINIT:-$DATA_MOUNT/etc-kubesolo/cloud-init.yaml}"
+if [ -f "$CLOUDINIT_FILE" ]; then
+    log "Cloud-init found: $CLOUDINIT_FILE"
+    # Phase 1: simple parsing — extract network stanza
+    # TODO: Replace with proper cloud-init parser (Go binary) in Phase 2
+    log_warn "Cloud-init network parsing not yet implemented — falling back to DHCP"
+fi
+
+# Fallback: DHCP on first non-loopback interface
+log "Configuring network via DHCP"
+
+# Bring up loopback
+ip link set lo up
+ip addr add 127.0.0.1/8 dev lo
+
+# Find first ethernet interface
+ETH_DEV=""
+for iface in /sys/class/net/*; do
+    iface="$(basename "$iface")"
+    case "$iface" in
+        lo|docker*|veth*|br*|cni*) continue ;;
+    esac
+    ETH_DEV="$iface"
+    break
+done
+
+if [ -z "$ETH_DEV" ]; then
+    log_err "No network interface found"
+    return 1
+fi
+
+log "Using interface: $ETH_DEV"
+ip link set "$ETH_DEV" up
+
+# Run DHCP client (BusyBox udhcpc)
+if command -v udhcpc >/dev/null 2>&1; then
+    udhcpc -i "$ETH_DEV" -s /usr/share/udhcpc/default.script \
+        -t 10 -T 3 -A 5 -b -q 2>/dev/null || {
+        log_err "DHCP failed on $ETH_DEV"
+        return 1
+    }
+elif command -v dhcpcd >/dev/null 2>&1; then
+    dhcpcd "$ETH_DEV" || {
+        log_err "DHCP failed on $ETH_DEV"
+        return 1
+    }
+else
+    log_err "No DHCP client available (need udhcpc or dhcpcd)"
+    return 1
+fi
+
+log_ok "Network configured on $ETH_DEV"
--- a/init/lib/60-hostname.sh
+++ b/init/lib/60-hostname.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# 60-hostname.sh — Set system hostname
+
+if [ -f "$DATA_MOUNT/etc-kubesolo/hostname" ]; then
+    HOSTNAME="$(cat "$DATA_MOUNT/etc-kubesolo/hostname")"
+elif [ -f /etc/kubesolo/hostname ]; then
+    HOSTNAME="$(cat /etc/kubesolo/hostname)"
+else
+    # Generate hostname from MAC address of primary interface
+    MAC_SUFFIX=""
+    for iface in /sys/class/net/*; do
+        iface="$(basename "$iface")"
+        case "$iface" in lo|docker*|veth*|br*|cni*) continue ;; esac
+        MAC_SUFFIX="$(cat "/sys/class/net/$iface/address" 2>/dev/null | tr -d ':' | tail -c 7)"
+        break
+    done
+    HOSTNAME="kubesolo-${MAC_SUFFIX:-unknown}"
+fi
+
+hostname "$HOSTNAME"
+echo "$HOSTNAME" > /etc/hostname
+echo "127.0.0.1 $HOSTNAME" >> /etc/hosts
+
+log_ok "Hostname set to: $HOSTNAME"
--- a/init/lib/70-clock.sh
+++ b/init/lib/70-clock.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+# 70-clock.sh — Set system clock (best-effort NTP or hwclock)
+
+# Try hardware clock first
+if command -v hwclock >/dev/null 2>&1; then
+    hwclock -s 2>/dev/null && log "Clock set from hardware clock" && return 0
+fi
+
+# Try NTP (one-shot, non-blocking)
+if command -v ntpd >/dev/null 2>&1; then
+    ntpd -n -q -p pool.ntp.org >/dev/null 2>&1 &
+    log "NTP sync started in background"
+elif command -v ntpdate >/dev/null 2>&1; then
+    ntpdate -u pool.ntp.org >/dev/null 2>&1 &
+    log "NTP sync started in background"
+else
+    log_warn "No NTP client available — clock may be inaccurate"
+    log_warn "K8s certificate validation may fail if clock is far off"
+fi
--- a/init/lib/80-containerd.sh
+++ b/init/lib/80-containerd.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+# 80-containerd.sh — Start containerd (bundled with KubeSolo)
+#
+# NOTE: KubeSolo typically manages containerd startup internally.
+# This stage ensures containerd prerequisites are met.
+# If KubeSolo handles containerd lifecycle, this stage may be a no-op.
+
+# Ensure containerd state directories exist
+mkdir -p /run/containerd
+mkdir -p /var/lib/containerd
+
+# Ensure CNI directories exist
+mkdir -p /etc/cni/net.d
+mkdir -p /opt/cni/bin
+
+# If containerd config doesn't exist, KubeSolo will use defaults
+# Only create a custom config if we need to override something
+if [ -f /etc/kubesolo/containerd-config.toml ]; then
+    log "Using custom containerd config from /etc/kubesolo/containerd-config.toml"
+fi
+
+log_ok "containerd prerequisites ready (KubeSolo manages containerd lifecycle)"
--- a/init/lib/90-kubesolo.sh
+++ b/init/lib/90-kubesolo.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+# 90-kubesolo.sh — Start KubeSolo (final init stage)
+#
+# This stage exec's KubeSolo as PID 1 (replacing init).
+# KubeSolo manages containerd, kubelet, API server, and all K8s components.
+
+KUBESOLO_BIN="/usr/local/bin/kubesolo"
+
+if [ ! -x "$KUBESOLO_BIN" ]; then
+    log_err "KubeSolo binary not found at $KUBESOLO_BIN"
+    return 1
+fi
+
+# Build KubeSolo command line
+KUBESOLO_ARGS="--path /var/lib/kubesolo --local-storage true"
+
+# Add extra SANs if hostname resolves
+HOSTNAME="$(hostname)"
+if [ -n "$HOSTNAME" ]; then
+    KUBESOLO_ARGS="$KUBESOLO_ARGS --apiserver-extra-sans $HOSTNAME"
+fi
+
+# Add any extra flags from boot parameters
+if [ -n "$KUBESOLO_EXTRA_FLAGS" ]; then
+    KUBESOLO_ARGS="$KUBESOLO_ARGS $KUBESOLO_EXTRA_FLAGS"
+fi
+
+# Add flags from persistent config file
+if [ -f /etc/kubesolo/extra-flags ]; then
+    KUBESOLO_ARGS="$KUBESOLO_ARGS $(cat /etc/kubesolo/extra-flags)"
+fi
+
+log "Starting KubeSolo: $KUBESOLO_BIN $KUBESOLO_ARGS"
+log "Kubeconfig will be at: /var/lib/kubesolo/pki/admin/admin.kubeconfig"
+
+# exec replaces this init process — KubeSolo becomes PID 1
+# shellcheck disable=SC2086
+exec $KUBESOLO_BIN $KUBESOLO_ARGS
--- a/init/lib/functions.sh
+++ b/init/lib/functions.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+# functions.sh — Shared utility functions for KubeSolo OS init
+# Sourced by /sbin/init before running stages
+# POSIX sh only — must work with BusyBox ash
+
+# Wait for a block device to appear
+wait_for_device() {
+    dev="$1"
+    timeout="${2:-30}"
+    i=0
+    while [ "$i" -lt "$timeout" ]; do
+        [ -b "$dev" ] && return 0
+        sleep 1
+        i=$((i + 1))
+    done
+    return 1
+}
+
+# Wait for a file to appear
+wait_for_file() {
+    path="$1"
+    timeout="${2:-30}"
+    i=0
+    while [ "$i" -lt "$timeout" ]; do
+        [ -f "$path" ] && return 0
+        sleep 1
+        i=$((i + 1))
+    done
+    return 1
+}
+
+# Get IP address of an interface (POSIX-safe, no grep -P)
+get_iface_ip() {
+    iface="$1"
+    ip -4 addr show "$iface" 2>/dev/null | \
+        sed -n 's/.*inet \([0-9.]*\).*/\1/p' | head -1
+}
+
+# Check if running in a VM (useful for adjusting timeouts)
+is_virtual() {
+    [ -d /sys/class/dmi/id ] && \
+        grep -qi -e 'qemu' -e 'kvm' -e 'vmware' -e 'virtualbox' -e 'xen' -e 'hyperv' \
+            /sys/class/dmi/id/sys_vendor 2>/dev/null
+}
+
+# Resolve a LABEL= or UUID= device spec to a block device path
+resolve_device() {
+    spec="$1"
+    case "$spec" in
+        LABEL=*) blkid -L "${spec#LABEL=}" 2>/dev/null ;;
+        UUID=*)  blkid -U "${spec#UUID=}" 2>/dev/null ;;
+        *)       echo "$spec" ;;
+    esac
+}
+
+# Write a key=value pair to a simple config file
+config_set() {
+    file="$1" key="$2" value="$3"
+    if grep -q "^${key}=" "$file" 2>/dev/null; then
+        sed -i "s|^${key}=.*|${key}=${value}|" "$file"
+    else
+        echo "${key}=${value}" >> "$file"
+    fi
+}
+
+# Read a value from a simple key=value config file
+config_get() {
+    file="$1" key="$2" default="${3:-}"
+    if [ -f "$file" ]; then
+        value=$(sed -n "s/^${key}=//p" "$file" | tail -1)
+        echo "${value:-$default}"
+    else
+        echo "$default"
+    fi
+}