Compare commits
20 Commits
456aa8eb5b
...
v0.2.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 4e3f1d6cf0 | |||
| 6ff77c4482 | |||
| a2764218fc | |||
| 2ba816bf6e | |||
| 65dcddb47e | |||
| ba4812f637 | |||
| 09dcea84ef | |||
| a4e719ba0e | |||
| 61bd28c692 | |||
| 4fc078f7a3 | |||
| 6c15ba7776 | |||
| 958524e6d8 | |||
| efc7f80b65 | |||
| 7abf0e0c04 | |||
| 60d0edaf84 | |||
| f3d86e4d8f | |||
| 04a5179533 | |||
| d9ac58418d | |||
| 36311ed4f4 | |||
| 39732488ef |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -23,3 +23,6 @@ Thumbs.db
|
|||||||
# Go
|
# Go
|
||||||
update/update-agent
|
update/update-agent
|
||||||
cloud-init/cloud-init-parser
|
cloud-init/cloud-init-parser
|
||||||
|
|
||||||
|
# Local docs (not tracked)
|
||||||
|
TINYCORE-MODIFICATIONS.md
|
||||||
|
|||||||
100
CHANGELOG.md
Normal file
100
CHANGELOG.md
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to KubeSolo OS are documented in this file.
|
||||||
|
|
||||||
|
Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
|
versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [0.2.0] - 2026-02-12
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Cloud-init: support all documented KubeSolo CLI flags (`--local-storage-shared-path`, `--debug`, `--pprof-server`, `--portainer-edge-id`, `--portainer-edge-key`, `--portainer-edge-async`)
|
||||||
|
- Cloud-init: `full-config.yaml` example showing all supported parameters
|
||||||
|
- Cloud-init: KubeSolo configuration reference table in docs/cloud-init.md
|
||||||
|
- Security hardening: mount hardening, sysctl, kernel module lock, AppArmor profiles
|
||||||
|
- ARM64 Raspberry Pi support with A/B boot via tryboot
|
||||||
|
- BootEnv abstraction for GRUB and RPi boot environments
|
||||||
|
- Go 1.25.5 installed on host for native builds
|
||||||
|
|
||||||
|
## [0.1.0] - 2026-02-12
|
||||||
|
|
||||||
|
First release with all 5 design-doc phases complete. ISO boots and runs K8s pods.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
#### Custom Kernel
|
||||||
|
- Custom kernel build (6.18.2-tinycore64) with container-critical configs
|
||||||
|
- Added CONFIG_CGROUP_BPF, CONFIG_DEVTMPFS, CONFIG_DEVTMPFS_MOUNT, CONFIG_MEMCG, CONFIG_CFS_BANDWIDTH
|
||||||
|
- Stripped unnecessary subsystems (sound, GPU, wireless, Bluetooth, etc.)
|
||||||
|
- Selective kernel module install — only modules.list + transitive deps in initramfs
|
||||||
|
|
||||||
|
#### Init System (Phase 1)
|
||||||
|
- POSIX sh init system with staged boot (00-early-mount through 90-kubesolo)
|
||||||
|
- switch_root from initramfs to SquashFS root
|
||||||
|
- Persistent data partition mount with bind-mounts for K8s state
|
||||||
|
- Kernel module loading, sysctl tuning, network, hostname, NTP
|
||||||
|
- Emergency shell fallback on boot failure
|
||||||
|
- Device node creation via mknod fallback from sysfs
|
||||||
|
|
||||||
|
#### Cloud-Init (Phase 2)
|
||||||
|
- Go-based cloud-init parser (~2.7 MB static binary)
|
||||||
|
- Network configuration: DHCP and static IP modes
|
||||||
|
- Hostname and machine-id generation
|
||||||
|
- KubeSolo configuration (node-name, extra flags)
|
||||||
|
- Portainer Edge Agent integration via K8s manifest injection
|
||||||
|
- Persistent config saved to /mnt/data/ for next-boot fast path
|
||||||
|
- 22 Go tests
|
||||||
|
|
||||||
|
#### A/B Atomic Updates (Phase 3)
|
||||||
|
- 4-partition GPT disk image: EFI + System A + System B + Data
|
||||||
|
- GRUB 2 bootloader with A/B slot selection and boot counter rollback
|
||||||
|
- Go update agent (~6.0 MB static binary) with check, apply, activate, rollback commands
|
||||||
|
- Health check: containerd + K8s API + node Ready verification
|
||||||
|
- Update server protocol: HTTP serving latest.json + image files
|
||||||
|
- K8s CronJob for automated update checks (every 6 hours)
|
||||||
|
- Zero external Go dependencies — uses kubectl/ctr exec commands
|
||||||
|
|
||||||
|
#### Production Hardening (Phase 4)
|
||||||
|
- Ed25519 image signing with pure Go stdlib (zero external deps)
|
||||||
|
- Key generation, signing, and verification CLI commands
|
||||||
|
- Portainer Edge Agent deployment via cloud-init
|
||||||
|
- SSH extension injection for debugging (hack/inject-ssh.sh)
|
||||||
|
- Boot time and resource usage benchmarks
|
||||||
|
- Deployment guide documentation
|
||||||
|
|
||||||
|
#### Distribution & Fleet Management (Phase 5)
|
||||||
|
- Gitea Actions CI/CD (test + build + shellcheck on push, release on tags)
|
||||||
|
- OCI container image packaging (scratch-based)
|
||||||
|
- Prometheus metrics endpoint (zero-dependency text exposition format)
|
||||||
|
- USB provisioning script with cloud-init injection
|
||||||
|
- ARM64 cross-compilation support
|
||||||
|
|
||||||
|
#### Build System
|
||||||
|
- Makefile with full build orchestration
|
||||||
|
- Dockerized reproducible builds (build/Dockerfile.builder)
|
||||||
|
- Component fetching with version pinning
|
||||||
|
- ISO and raw disk image creation
|
||||||
|
- Fast rebuild path (`make quick`)
|
||||||
|
|
||||||
|
#### Documentation
|
||||||
|
- Architecture design document
|
||||||
|
- Boot flow reference
|
||||||
|
- A/B update flow reference
|
||||||
|
- Cloud-init configuration reference
|
||||||
|
- Deployment and operations guide
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Replaced `grep -oP` with POSIX-safe `sed` in functions.sh (BusyBox compatibility)
|
||||||
|
- Replaced `grep -qiE` with `grep -qi -e` pattern (POSIX compliance)
|
||||||
|
- Fixed KVM flag handling in dev-vm.sh (bash array context)
|
||||||
|
- Added iptables table pre-initialization before kube-proxy start (nf_tables issue)
|
||||||
|
- Added /dev/kmsg and /etc/machine-id creation for kubelet
|
||||||
|
- Added CA certificates bundle to initramfs (containerd TLS verification for Docker Hub)
|
||||||
|
- Added DNS fallback (10.0.2.3 + 8.8.8.8) when DHCP client doesn't populate resolv.conf
|
||||||
|
- Added headless Service to Portainer Edge Agent manifest (agent peer discovery DNS)
|
||||||
|
- Added kubesolo.edge_id/edge_key kernel boot parameters for Portainer Edge
|
||||||
|
- Added auto-format of unformatted data disks on first boot
|
||||||
|
- Rewrote dev-vm.sh for macOS: bsdtar ISO extraction, Homebrew mkfs.ext4 detection, direct kernel boot, TCG acceleration, port 8080 forwarding
|
||||||
|
- Kubeconfig now served via HTTP on port 8080 (serial console truncates base64 lines)
|
||||||
|
- Added 127.0.0.1 and 10.0.2.15 to API server SANs for QEMU port forwarding
|
||||||
|
- dev-vm.sh now works on Linux: fallback ISO extraction via isoinfo or loop mount, KVM auto-detection, platform-aware error messages
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2025 Anthony De Lorenzo
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
64
Makefile
64
Makefile
@@ -1,9 +1,10 @@
|
|||||||
.PHONY: all fetch build-cloudinit build-update-agent build-cross rootfs initramfs \
|
.PHONY: all fetch kernel build-cloudinit build-update-agent build-cross rootfs initramfs \
|
||||||
iso disk-image oci-image \
|
iso disk-image oci-image rpi-image \
|
||||||
test-boot test-k8s test-persistence test-deploy test-storage test-all \
|
kernel-arm64 rootfs-arm64 \
|
||||||
test-cloudinit test-update-agent \
|
test-boot test-k8s test-persistence test-deploy test-storage test-security test-all \
|
||||||
|
test-boot-arm64 test-cloudinit test-update-agent \
|
||||||
bench-boot bench-resources \
|
bench-boot bench-resources \
|
||||||
dev-vm dev-vm-shell quick docker-build shellcheck \
|
dev-vm dev-vm-shell dev-vm-arm64 quick docker-build shellcheck \
|
||||||
kernel-audit clean distclean help
|
kernel-audit clean distclean help
|
||||||
|
|
||||||
SHELL := /bin/bash
|
SHELL := /bin/bash
|
||||||
@@ -30,6 +31,10 @@ fetch:
|
|||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Build stages
|
# Build stages
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
kernel:
|
||||||
|
@echo "==> Building custom kernel (CONFIG_CGROUP_BPF=y)..."
|
||||||
|
$(BUILD_DIR)/scripts/build-kernel.sh
|
||||||
|
|
||||||
build-cloudinit:
|
build-cloudinit:
|
||||||
@echo "==> Building cloud-init binary..."
|
@echo "==> Building cloud-init binary..."
|
||||||
$(BUILD_DIR)/scripts/build-cloudinit.sh
|
$(BUILD_DIR)/scripts/build-cloudinit.sh
|
||||||
@@ -38,7 +43,7 @@ build-update-agent:
|
|||||||
@echo "==> Building update agent..."
|
@echo "==> Building update agent..."
|
||||||
$(BUILD_DIR)/scripts/build-update-agent.sh
|
$(BUILD_DIR)/scripts/build-update-agent.sh
|
||||||
|
|
||||||
rootfs: fetch build-cloudinit build-update-agent
|
rootfs: fetch kernel build-cloudinit build-update-agent
|
||||||
@echo "==> Preparing rootfs..."
|
@echo "==> Preparing rootfs..."
|
||||||
$(BUILD_DIR)/scripts/extract-core.sh
|
$(BUILD_DIR)/scripts/extract-core.sh
|
||||||
$(BUILD_DIR)/scripts/inject-kubesolo.sh
|
$(BUILD_DIR)/scripts/inject-kubesolo.sh
|
||||||
@@ -67,6 +72,26 @@ build-cross:
|
|||||||
@echo "==> Cross-compiling for amd64 + arm64..."
|
@echo "==> Cross-compiling for amd64 + arm64..."
|
||||||
$(BUILD_DIR)/scripts/build-cross.sh
|
$(BUILD_DIR)/scripts/build-cross.sh
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# ARM64 Raspberry Pi targets
|
||||||
|
# =============================================================================
|
||||||
|
kernel-arm64:
|
||||||
|
@echo "==> Building ARM64 kernel for Raspberry Pi..."
|
||||||
|
$(BUILD_DIR)/scripts/build-kernel-arm64.sh
|
||||||
|
|
||||||
|
rootfs-arm64: build-cross
|
||||||
|
@echo "==> Preparing ARM64 rootfs..."
|
||||||
|
TARGET_ARCH=arm64 $(BUILD_DIR)/scripts/fetch-components.sh
|
||||||
|
TARGET_ARCH=arm64 $(BUILD_DIR)/scripts/extract-core.sh
|
||||||
|
TARGET_ARCH=arm64 $(BUILD_DIR)/scripts/inject-kubesolo.sh
|
||||||
|
@echo "==> Packing ARM64 initramfs..."
|
||||||
|
$(BUILD_DIR)/scripts/pack-initramfs.sh
|
||||||
|
|
||||||
|
rpi-image: rootfs-arm64 kernel-arm64
|
||||||
|
@echo "==> Creating Raspberry Pi SD card image..."
|
||||||
|
$(BUILD_DIR)/scripts/create-rpi-image.sh
|
||||||
|
@echo "==> Built: $(OUTPUT_DIR)/$(OS_NAME)-$(VERSION).rpi.img"
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Kernel validation
|
# Kernel validation
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
@@ -97,6 +122,14 @@ test-storage: iso
|
|||||||
@echo "==> Testing local storage provisioning..."
|
@echo "==> Testing local storage provisioning..."
|
||||||
test/integration/test-local-storage.sh $(OUTPUT_DIR)/$(OS_NAME)-$(VERSION).iso
|
test/integration/test-local-storage.sh $(OUTPUT_DIR)/$(OS_NAME)-$(VERSION).iso
|
||||||
|
|
||||||
|
test-security: iso
|
||||||
|
@echo "==> Testing security hardening..."
|
||||||
|
test/integration/test-security-hardening.sh $(OUTPUT_DIR)/$(OS_NAME)-$(VERSION).iso
|
||||||
|
|
||||||
|
test-boot-arm64:
|
||||||
|
@echo "==> Testing ARM64 boot in QEMU..."
|
||||||
|
test/qemu/test-boot-arm64.sh
|
||||||
|
|
||||||
test-all: test-boot test-k8s test-persistence
|
test-all: test-boot test-k8s test-persistence
|
||||||
|
|
||||||
# Cloud-init Go tests
|
# Cloud-init Go tests
|
||||||
@@ -159,6 +192,10 @@ dev-vm-debug: iso
|
|||||||
@echo "==> Launching dev VM (debug mode)..."
|
@echo "==> Launching dev VM (debug mode)..."
|
||||||
hack/dev-vm.sh $(OUTPUT_DIR)/$(OS_NAME)-$(VERSION).iso --debug
|
hack/dev-vm.sh $(OUTPUT_DIR)/$(OS_NAME)-$(VERSION).iso --debug
|
||||||
|
|
||||||
|
dev-vm-arm64:
|
||||||
|
@echo "==> Launching ARM64 dev VM..."
|
||||||
|
hack/dev-vm-arm64.sh
|
||||||
|
|
||||||
# Fast rebuild: only repack initramfs + ISO (skip fetch/extract)
|
# Fast rebuild: only repack initramfs + ISO (skip fetch/extract)
|
||||||
quick:
|
quick:
|
||||||
@echo "==> Quick rebuild (repack + ISO only)..."
|
@echo "==> Quick rebuild (repack + ISO only)..."
|
||||||
@@ -176,7 +213,7 @@ docker-build:
|
|||||||
docker run --rm --privileged \
|
docker run --rm --privileged \
|
||||||
-v $(PWD)/$(OUTPUT_DIR):/output \
|
-v $(PWD)/$(OUTPUT_DIR):/output \
|
||||||
-v $(PWD)/$(CACHE_DIR):/cache \
|
-v $(PWD)/$(CACHE_DIR):/cache \
|
||||||
kubesolo-os-builder make iso OUTPUT_DIR=/output CACHE_DIR=/cache
|
kubesolo-os-builder iso OUTPUT_DIR=/output CACHE_DIR=/cache
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Cleanup
|
# Cleanup
|
||||||
@@ -195,8 +232,9 @@ distclean: clean
|
|||||||
help:
|
help:
|
||||||
@echo "KubeSolo OS Build System (v$(VERSION))"
|
@echo "KubeSolo OS Build System (v$(VERSION))"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Build targets:"
|
@echo "Build targets (x86_64):"
|
||||||
@echo " make fetch Download Tiny Core ISO, KubeSolo, dependencies"
|
@echo " make fetch Download Tiny Core ISO, KubeSolo, dependencies"
|
||||||
|
@echo " make kernel Build custom kernel with CONFIG_CGROUP_BPF=y"
|
||||||
@echo " make build-cloudinit Build cloud-init Go binary"
|
@echo " make build-cloudinit Build cloud-init Go binary"
|
||||||
@echo " make build-update-agent Build update agent Go binary"
|
@echo " make build-update-agent Build update agent Go binary"
|
||||||
@echo " make rootfs Extract + prepare rootfs with KubeSolo"
|
@echo " make rootfs Extract + prepare rootfs with KubeSolo"
|
||||||
@@ -208,25 +246,33 @@ help:
|
|||||||
@echo " make quick Fast rebuild (re-inject + repack + ISO only)"
|
@echo " make quick Fast rebuild (re-inject + repack + ISO only)"
|
||||||
@echo " make docker-build Reproducible build inside Docker"
|
@echo " make docker-build Reproducible build inside Docker"
|
||||||
@echo ""
|
@echo ""
|
||||||
|
@echo "Build targets (ARM64 Raspberry Pi):"
|
||||||
|
@echo " make kernel-arm64 Build ARM64 kernel from raspberrypi/linux"
|
||||||
|
@echo " make rootfs-arm64 Extract + prepare ARM64 rootfs from piCore64"
|
||||||
|
@echo " make rpi-image Create Raspberry Pi SD card image with A/B partitions"
|
||||||
|
@echo ""
|
||||||
@echo "Test targets:"
|
@echo "Test targets:"
|
||||||
@echo " make test-boot Boot ISO in QEMU, verify boot success"
|
@echo " make test-boot Boot ISO in QEMU, verify boot success"
|
||||||
@echo " make test-k8s Boot + verify K8s node reaches Ready"
|
@echo " make test-k8s Boot + verify K8s node reaches Ready"
|
||||||
@echo " make test-persist Reboot disk image, verify state persists"
|
@echo " make test-persist Reboot disk image, verify state persists"
|
||||||
@echo " make test-deploy Deploy nginx pod, verify Running"
|
@echo " make test-deploy Deploy nginx pod, verify Running"
|
||||||
@echo " make test-storage Test PVC with local-path provisioner"
|
@echo " make test-storage Test PVC with local-path provisioner"
|
||||||
|
@echo " make test-security Verify security hardening (AppArmor, sysctl, mounts)"
|
||||||
@echo " make test-cloudinit Run cloud-init Go unit tests"
|
@echo " make test-cloudinit Run cloud-init Go unit tests"
|
||||||
@echo " make test-update-agent Run update agent Go unit tests"
|
@echo " make test-update-agent Run update agent Go unit tests"
|
||||||
@echo " make test-update A/B update cycle integration test"
|
@echo " make test-update A/B update cycle integration test"
|
||||||
@echo " make test-rollback Forced rollback integration test"
|
@echo " make test-rollback Forced rollback integration test"
|
||||||
|
@echo " make test-boot-arm64 ARM64 boot test in QEMU aarch64"
|
||||||
@echo " make test-all Run core tests (boot + k8s + persistence)"
|
@echo " make test-all Run core tests (boot + k8s + persistence)"
|
||||||
@echo " make test-integ Run full integration suite"
|
@echo " make test-integ Run full integration suite"
|
||||||
@echo " make bench-boot Benchmark boot performance (3 runs)"
|
@echo " make bench-boot Benchmark boot performance (3 runs)"
|
||||||
@echo " make bench-resources Benchmark resource usage (requires running VM)"
|
@echo " make bench-resources Benchmark resource usage (requires running VM)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Dev targets:"
|
@echo "Dev targets:"
|
||||||
@echo " make dev-vm Launch interactive QEMU VM"
|
@echo " make dev-vm Launch interactive QEMU VM (x86_64)"
|
||||||
@echo " make dev-vm-shell Launch QEMU VM -> emergency shell"
|
@echo " make dev-vm-shell Launch QEMU VM -> emergency shell"
|
||||||
@echo " make dev-vm-debug Launch QEMU VM with debug logging"
|
@echo " make dev-vm-debug Launch QEMU VM with debug logging"
|
||||||
|
@echo " make dev-vm-arm64 Launch ARM64 QEMU VM"
|
||||||
@echo " make kernel-audit Check kernel config against requirements"
|
@echo " make kernel-audit Check kernel config against requirements"
|
||||||
@echo " make shellcheck Lint all shell scripts"
|
@echo " make shellcheck Lint all shell scripts"
|
||||||
@echo ""
|
@echo ""
|
||||||
|
|||||||
200
README.md
200
README.md
@@ -2,18 +2,23 @@
|
|||||||
|
|
||||||
An immutable, bootable Linux distribution purpose-built for [KubeSolo](https://github.com/portainer/kubesolo) — Portainer's ultra-lightweight single-node Kubernetes.
|
An immutable, bootable Linux distribution purpose-built for [KubeSolo](https://github.com/portainer/kubesolo) — Portainer's ultra-lightweight single-node Kubernetes.
|
||||||
|
|
||||||
> **Status:** Phase 1 — Proof of Concept
|
> **Status:** All 6 phases complete. Boots and runs K8s workloads. Portainer Edge Agent tested and connected.
|
||||||
|
|
||||||
## What is this?
|
## What is this?
|
||||||
|
|
||||||
KubeSolo OS combines **Tiny Core Linux** (~11 MB) with **KubeSolo** (single-binary Kubernetes) to create an appliance-like K8s node that:
|
KubeSolo OS combines **Tiny Core Linux** (~11 MB) with **KubeSolo** (single-binary Kubernetes) to create an appliance-like K8s node that:
|
||||||
|
|
||||||
- Boots to a functional Kubernetes cluster in ~30 seconds
|
- Boots to a functional Kubernetes cluster in ~35 seconds
|
||||||
- Runs entirely from RAM with a read-only SquashFS root
|
- Runs entirely from RAM with a read-only SquashFS root
|
||||||
- Persists K8s state across reboots via a dedicated data partition
|
- Persists K8s state across reboots via a dedicated data partition
|
||||||
- Targets < 100 MB total image size (OS + K8s)
|
- Uses a custom kernel (6.18.2-tinycore64) optimized for containers
|
||||||
|
- Supports first-boot configuration via cloud-init YAML
|
||||||
|
- Performs atomic A/B updates with automatic GRUB-based rollback
|
||||||
|
- Signs update images with Ed25519 for integrity verification
|
||||||
|
- Exposes Prometheus metrics for monitoring
|
||||||
|
- Integrates with Portainer Edge for fleet management
|
||||||
|
- Ships as ISO, raw disk image, or OCI container
|
||||||
- Requires no SSH, no package manager, no writable system files
|
- Requires no SSH, no package manager, no writable system files
|
||||||
- Supports atomic A/B updates with automatic rollback (Phase 3)
|
|
||||||
|
|
||||||
**Target use cases:** IoT/IIoT edge, air-gapped deployments, single-node K8s appliances, kiosk/POS systems, resource-constrained hardware.
|
**Target use cases:** IoT/IIoT edge, air-gapped deployments, single-node K8s appliances, kiosk/POS systems, resource-constrained hardware.
|
||||||
|
|
||||||
@@ -23,64 +28,213 @@ KubeSolo OS combines **Tiny Core Linux** (~11 MB) with **KubeSolo** (single-bina
|
|||||||
# Fetch Tiny Core ISO + KubeSolo binary
|
# Fetch Tiny Core ISO + KubeSolo binary
|
||||||
make fetch
|
make fetch
|
||||||
|
|
||||||
|
# Build custom kernel (first time only, ~25 min, cached)
|
||||||
|
make kernel
|
||||||
|
|
||||||
|
# Build Go binaries
|
||||||
|
make build-cloudinit build-update-agent
|
||||||
|
|
||||||
# Build bootable ISO
|
# Build bootable ISO
|
||||||
make iso
|
make rootfs initramfs iso
|
||||||
|
|
||||||
# Test in QEMU
|
# Test in QEMU
|
||||||
make dev-vm
|
make dev-vm
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Or build everything at once inside Docker:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make docker-build
|
||||||
|
```
|
||||||
|
|
||||||
|
After boot, retrieve the kubeconfig and manage your cluster from the host:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s http://localhost:8080 > ~/.kube/kubesolo-config
|
||||||
|
export KUBECONFIG=~/.kube/kubesolo-config
|
||||||
|
kubectl get nodes
|
||||||
|
```
|
||||||
|
|
||||||
|
### Portainer Edge Agent
|
||||||
|
|
||||||
|
Pass Edge credentials via boot parameters:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./hack/dev-vm.sh --edge-id=YOUR_EDGE_ID --edge-key=YOUR_EDGE_KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
Or configure via [cloud-init YAML](cloud-init/examples/portainer-edge.yaml).
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
**Build host:**
|
**Build host:**
|
||||||
- Linux x86_64 with root/sudo (for loop mounts)
|
- Linux x86_64 with root/sudo (for loop mounts)
|
||||||
|
- Go 1.22+ (for cloud-init and update agent)
|
||||||
- Tools: `cpio`, `gzip`, `wget`, `curl`, `syslinux` (or use `make docker-build`)
|
- Tools: `cpio`, `gzip`, `wget`, `curl`, `syslinux` (or use `make docker-build`)
|
||||||
|
|
||||||
**Runtime:**
|
**Runtime:**
|
||||||
- x86_64 hardware or VM
|
- x86_64 hardware or VM (ARM64 cross-compilation available)
|
||||||
- 512 MB RAM minimum (1 GB+ recommended)
|
- 512 MB RAM minimum (1 GB+ recommended)
|
||||||
- 8 GB disk (for persistent data partition)
|
- 8 GB disk (for persistent data partition)
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
Boot Media → Kernel + Initramfs (kubesolo-os.gz)
|
Boot Media (ISO or Disk Image)
|
||||||
│
|
│
|
||||||
├── SquashFS root (read-only, in RAM)
|
├── GRUB 2 bootloader (A/B slot selection, rollback counter)
|
||||||
|
│
|
||||||
|
└── Kernel + Initramfs (kubesolo-os.gz)
|
||||||
|
│
|
||||||
|
├── switch_root → SquashFS root (read-only, in RAM)
|
||||||
├── Persistent data partition (ext4, bind-mounted)
|
├── Persistent data partition (ext4, bind-mounted)
|
||||||
│ ├── /var/lib/kubesolo (K8s state, certs, SQLite)
|
│ ├── /var/lib/kubesolo (K8s state, certs, SQLite)
|
||||||
│ ├── /var/lib/containerd (container images)
|
│ ├── /var/lib/containerd (container images)
|
||||||
│ └── /etc/kubesolo (node configuration)
|
│ └── /etc/kubesolo (node configuration)
|
||||||
├── Custom init (POSIX sh, staged boot)
|
├── Custom init (POSIX sh, staged boot 00→90)
|
||||||
└── KubeSolo (exec replaces init as PID 1)
|
│ └── Stage 45: cloud-init (Go binary)
|
||||||
|
├── containerd (bundled with KubeSolo)
|
||||||
|
└── KubeSolo (single-binary K8s)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Partition Layout (Disk Image)
|
||||||
|
|
||||||
|
```
|
||||||
|
GPT Disk (minimum 8 GB):
|
||||||
|
Part 1: EFI/Boot (256 MB, FAT32) — GRUB + A/B boot logic
|
||||||
|
Part 2: System A (512 MB, ext4) — vmlinuz + kubesolo-os.gz (active)
|
||||||
|
Part 3: System B (512 MB, ext4) — vmlinuz + kubesolo-os.gz (passive)
|
||||||
|
Part 4: Data (remaining, ext4) — persistent K8s state
|
||||||
```
|
```
|
||||||
|
|
||||||
See [docs/design/kubesolo-os-design.md](docs/design/kubesolo-os-design.md) for the full architecture document.
|
See [docs/design/kubesolo-os-design.md](docs/design/kubesolo-os-design.md) for the full architecture document.
|
||||||
|
|
||||||
|
## Custom Kernel
|
||||||
|
|
||||||
|
The stock Tiny Core 17.0 kernel lacks several configs required for containers. KubeSolo OS builds a custom kernel (6.18.2-tinycore64) that adds:
|
||||||
|
|
||||||
|
- `CONFIG_CGROUP_BPF` — cgroup v2 device control via BPF
|
||||||
|
- `CONFIG_DEVTMPFS` / `CONFIG_DEVTMPFS_MOUNT` — automatic /dev node creation
|
||||||
|
- `CONFIG_MEMCG` — memory cgroup controller
|
||||||
|
- `CONFIG_CFS_BANDWIDTH` — CPU bandwidth throttling
|
||||||
|
|
||||||
|
Unnecessary subsystems (sound, GPU, wireless, Bluetooth, etc.) are stripped to keep the kernel minimal. Build is cached in `build/cache/custom-kernel/`.
|
||||||
|
|
||||||
|
## Cloud-Init
|
||||||
|
|
||||||
|
First-boot configuration via a simple YAML schema. All [documented KubeSolo flags](https://www.kubesolo.io/documentation#install) are supported:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
hostname: edge-node-01
|
||||||
|
network:
|
||||||
|
mode: static
|
||||||
|
address: 192.168.1.100/24
|
||||||
|
gateway: 192.168.1.1
|
||||||
|
dns:
|
||||||
|
- 8.8.8.8
|
||||||
|
kubesolo:
|
||||||
|
local-storage: true
|
||||||
|
local-storage-shared-path: "/mnt/shared"
|
||||||
|
apiserver-extra-sans:
|
||||||
|
- edge-node-01.local
|
||||||
|
debug: false
|
||||||
|
pprof-server: false
|
||||||
|
portainer-edge-id: "your-edge-id"
|
||||||
|
portainer-edge-key: "your-edge-key"
|
||||||
|
portainer-edge-async: true
|
||||||
|
```
|
||||||
|
|
||||||
|
See [docs/cloud-init.md](docs/cloud-init.md) and the [examples](cloud-init/examples/).
|
||||||
|
|
||||||
|
## Atomic Updates
|
||||||
|
|
||||||
|
A/B partition scheme with GRUB boot counter for automatic rollback:
|
||||||
|
|
||||||
|
1. Update agent downloads new image to passive partition
|
||||||
|
2. GRUB boots new partition with `boot_counter=3`
|
||||||
|
3. Health check verifies containerd + K8s API + node Ready → sets `boot_success=1`
|
||||||
|
4. On 3 consecutive boot failures, GRUB auto-rolls back to previous slot
|
||||||
|
|
||||||
|
Updates can be signed with Ed25519 for integrity verification. A K8s CronJob checks for updates every 6 hours.
|
||||||
|
|
||||||
|
See [docs/update-flow.md](docs/update-flow.md).
|
||||||
|
|
||||||
|
## Monitoring
|
||||||
|
|
||||||
|
The update agent exposes Prometheus metrics on port 9100:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubesolo-update metrics --listen :9100
|
||||||
|
```
|
||||||
|
|
||||||
|
Metrics include: `kubesolo_os_info`, `boot_success`, `boot_counter`, `uptime_seconds`, `update_available`, `memory_total_bytes`, `memory_available_bytes`.
|
||||||
|
|
||||||
## Project Structure
|
## Project Structure
|
||||||
|
|
||||||
```
|
```
|
||||||
├── CLAUDE.md # AI-assisted development instructions
|
|
||||||
├── Makefile # Build orchestration
|
├── Makefile # Build orchestration
|
||||||
├── build/ # Build scripts, configs, rootfs overlays
|
├── build/ # Build scripts, kernel config, rootfs overlays
|
||||||
|
│ └── scripts/
|
||||||
|
│ ├── build-kernel.sh # Custom kernel compilation
|
||||||
|
│ ├── fetch-components.sh # Download components
|
||||||
|
│ ├── create-iso.sh # Bootable ISO
|
||||||
|
│ ├── create-disk-image.sh # A/B partition disk image
|
||||||
|
│ └── create-oci-image.sh # OCI container image
|
||||||
├── init/ # Custom init system (POSIX sh)
|
├── init/ # Custom init system (POSIX sh)
|
||||||
├── update/ # Atomic update agent (Go, Phase 3)
|
│ ├── init.sh # Main init + switch_root
|
||||||
├── cloud-init/ # First-boot configuration (Phase 2)
|
│ └── lib/ # Staged boot scripts (00-90)
|
||||||
├── test/ # QEMU-based automated tests
|
├── cloud-init/ # Go cloud-init parser
|
||||||
├── hack/ # Developer utilities
|
├── update/ # Go atomic update agent
|
||||||
└── docs/ # Design documents
|
├── test/ # QEMU-based automated tests + benchmarks
|
||||||
|
├── hack/ # Developer utilities (dev-vm, SSH, USB)
|
||||||
|
├── docs/ # Documentation
|
||||||
|
│ ├── design/ # Architecture design document
|
||||||
|
│ ├── boot-flow.md # Boot sequence reference
|
||||||
|
│ ├── update-flow.md # A/B update reference
|
||||||
|
│ ├── cloud-init.md # Cloud-init configuration reference
|
||||||
|
│ └── deployment-guide.md # Deployment and operations guide
|
||||||
|
└── .gitea/workflows/ # CI/CD (Gitea Actions)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Make Targets
|
||||||
|
|
||||||
|
| Target | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `make fetch` | Download Tiny Core ISO + KubeSolo binary |
|
||||||
|
| `make kernel` | Build custom kernel (cached) |
|
||||||
|
| `make build-cloudinit` | Compile cloud-init Go binary |
|
||||||
|
| `make build-update-agent` | Compile update agent Go binary |
|
||||||
|
| `make rootfs` | Extract Tiny Core + inject KubeSolo |
|
||||||
|
| `make initramfs` | Pack initramfs (kubesolo-os.gz) |
|
||||||
|
| `make iso` | Create bootable ISO |
|
||||||
|
| `make disk-image` | Create A/B partition disk image |
|
||||||
|
| `make oci-image` | Package as OCI container |
|
||||||
|
| `make build-cross` | Cross-compile for amd64 + arm64 |
|
||||||
|
| `make docker-build` | Build everything in Docker |
|
||||||
|
| `make quick` | Fast rebuild (re-inject + repack + ISO) |
|
||||||
|
| `make dev-vm` | Launch QEMU dev VM (Linux + macOS) |
|
||||||
|
| `make test-all` | Run all tests |
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
- [Architecture Design](docs/design/kubesolo-os-design.md) — full research and technical specification
|
||||||
|
- [Boot Flow](docs/boot-flow.md) — boot sequence from GRUB to K8s Ready
|
||||||
|
- [Update Flow](docs/update-flow.md) — A/B atomic update mechanism
|
||||||
|
- [Cloud-Init](docs/cloud-init.md) — first-boot configuration reference
|
||||||
|
- [Deployment Guide](docs/deployment-guide.md) — installation, operations, troubleshooting
|
||||||
|
|
||||||
## Roadmap
|
## Roadmap
|
||||||
|
|
||||||
| Phase | Scope | Status |
|
| Phase | Scope | Status |
|
||||||
|-------|-------|--------|
|
|-------|-------|--------|
|
||||||
| 1 | PoC: boot Tiny Core + KubeSolo, verify K8s | 🚧 In Progress |
|
| 1 | PoC: boot Tiny Core + KubeSolo, verify K8s | Complete |
|
||||||
| 2 | Persistent storage, cloud-init, networking | Planned |
|
| 2 | Cloud-init Go parser, network, hostname | Complete |
|
||||||
| 3 | A/B atomic updates, GRUB, rollback | Planned |
|
| 3 | A/B atomic updates, GRUB, rollback agent | Complete |
|
||||||
| 4 | Production hardening, signing, Portainer Edge | Planned |
|
| 4 | Ed25519 signing, Portainer Edge, SSH extension | Complete |
|
||||||
| 5 | OCI distribution, ARM64, fleet management | Planned |
|
| 5 | CI/CD, OCI distribution, Prometheus metrics, ARM64 | Complete |
|
||||||
|
| 6 | Security hardening, AppArmor, ARM64 RPi support | Complete |
|
||||||
|
| - | Custom kernel build for container runtime fixes | Complete |
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
TBD
|
MIT License — see [LICENSE](LICENSE) for details.
|
||||||
|
|||||||
@@ -1,34 +1,60 @@
|
|||||||
FROM ubuntu:24.04
|
FROM --platform=linux/amd64 ubuntu:24.04
|
||||||
|
|
||||||
ENV DEBIAN_FRONTEND=noninteractive
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# Install build tools + kernel build dependencies
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
bash \
|
bash \
|
||||||
bsdtar \
|
bc \
|
||||||
|
bison \
|
||||||
|
build-essential \
|
||||||
|
ca-certificates \
|
||||||
cpio \
|
cpio \
|
||||||
curl \
|
curl \
|
||||||
dosfstools \
|
dosfstools \
|
||||||
|
dwarves \
|
||||||
e2fsprogs \
|
e2fsprogs \
|
||||||
fdisk \
|
fdisk \
|
||||||
|
file \
|
||||||
|
flex \
|
||||||
genisoimage \
|
genisoimage \
|
||||||
gzip \
|
gzip \
|
||||||
isolinux \
|
isolinux \
|
||||||
losetup \
|
iptables \
|
||||||
|
kmod \
|
||||||
|
libarchive-tools \
|
||||||
|
libelf-dev \
|
||||||
|
libssl-dev \
|
||||||
make \
|
make \
|
||||||
parted \
|
parted \
|
||||||
squashfs-tools \
|
squashfs-tools \
|
||||||
syslinux \
|
syslinux \
|
||||||
syslinux-common \
|
syslinux-common \
|
||||||
syslinux-utils \
|
syslinux-utils \
|
||||||
|
apparmor \
|
||||||
|
apparmor-utils \
|
||||||
|
gcc-aarch64-linux-gnu \
|
||||||
|
binutils-aarch64-linux-gnu \
|
||||||
|
git \
|
||||||
|
kpartx \
|
||||||
|
unzip \
|
||||||
wget \
|
wget \
|
||||||
xorriso \
|
xorriso \
|
||||||
xz-utils \
|
xz-utils \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Install Go (for building cloud-init and update agent)
|
||||||
|
ARG GO_VERSION=1.25.5
|
||||||
|
RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
|
||||||
|
| tar -C /usr/local -xzf -
|
||||||
|
ENV PATH="/usr/local/go/bin:${PATH}"
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
COPY . /build
|
COPY . /build
|
||||||
|
|
||||||
RUN chmod +x build/scripts/*.sh build/config/*.sh
|
RUN chmod +x build/scripts/*.sh build/config/*.sh \
|
||||||
|
&& chmod +x hack/*.sh 2>/dev/null || true \
|
||||||
|
&& chmod +x test/qemu/*.sh test/integration/*.sh test/kernel/*.sh 2>/dev/null || true
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/bin/make"]
|
ENTRYPOINT ["/usr/bin/make"]
|
||||||
CMD ["iso"]
|
CMD ["iso"]
|
||||||
|
|||||||
@@ -128,7 +128,12 @@ echo "Security:"
|
|||||||
check_config CONFIG_SECCOMP recommended "Seccomp (container security)"
|
check_config CONFIG_SECCOMP recommended "Seccomp (container security)"
|
||||||
check_config CONFIG_SECCOMP_FILTER recommended "Seccomp BPF filter"
|
check_config CONFIG_SECCOMP_FILTER recommended "Seccomp BPF filter"
|
||||||
check_config CONFIG_BPF_SYSCALL recommended "BPF syscall"
|
check_config CONFIG_BPF_SYSCALL recommended "BPF syscall"
|
||||||
check_config CONFIG_AUDIT recommended "Audit framework"
|
check_config CONFIG_AUDIT mandatory "Audit framework"
|
||||||
|
check_config CONFIG_AUDITSYSCALL mandatory "Audit system call events"
|
||||||
|
check_config CONFIG_SECURITY mandatory "Security framework"
|
||||||
|
check_config CONFIG_SECURITYFS mandatory "Security filesystem"
|
||||||
|
check_config CONFIG_SECURITY_APPARMOR mandatory "AppArmor LSM"
|
||||||
|
check_config CONFIG_SECURITY_NETWORK recommended "Network security hooks"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
# --- Crypto ---
|
# --- Crypto ---
|
||||||
|
|||||||
81
build/config/modules-arm64.list
Normal file
81
build/config/modules-arm64.list
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
# Kernel modules loaded at boot by init (ARM64 / Raspberry Pi)
|
||||||
|
# One module per line. Lines starting with # are ignored.
|
||||||
|
# Modules are loaded in order listed — dependencies must come first.
|
||||||
|
|
||||||
|
# Network device drivers (loaded early so interfaces are available)
|
||||||
|
# Note: no e1000/e1000e on ARM64 — those are x86 Intel NIC drivers
|
||||||
|
virtio_net
|
||||||
|
|
||||||
|
# Virtio support (for QEMU VMs — block, entropy)
|
||||||
|
virtio_blk
|
||||||
|
virtio_rng
|
||||||
|
|
||||||
|
# Raspberry Pi specific (USB Ethernet on Pi 4 is built-in, no module needed)
|
||||||
|
# Pi 5 uses PCIe ethernet, also typically built-in
|
||||||
|
|
||||||
|
# Filesystem — overlay (required for containerd)
|
||||||
|
overlay
|
||||||
|
|
||||||
|
# Netfilter dependencies (must load before conntrack)
|
||||||
|
nf_defrag_ipv4
|
||||||
|
nf_defrag_ipv6
|
||||||
|
|
||||||
|
# Netfilter / connection tracking (required for kube-proxy)
|
||||||
|
nf_conntrack
|
||||||
|
nf_nat
|
||||||
|
nf_conntrack_netlink
|
||||||
|
|
||||||
|
# nftables (modern iptables backend)
|
||||||
|
nf_tables
|
||||||
|
nft_compat
|
||||||
|
nft_chain_nat
|
||||||
|
nft_ct
|
||||||
|
nft_masq
|
||||||
|
nft_nat
|
||||||
|
nft_redir
|
||||||
|
|
||||||
|
# Netfilter xt match/target modules (used by kube-proxy iptables rules via nft_compat)
|
||||||
|
xt_conntrack
|
||||||
|
xt_MASQUERADE
|
||||||
|
xt_mark
|
||||||
|
xt_comment
|
||||||
|
xt_multiport
|
||||||
|
xt_nat
|
||||||
|
xt_addrtype
|
||||||
|
xt_connmark
|
||||||
|
xt_REDIRECT
|
||||||
|
xt_recent
|
||||||
|
xt_statistic
|
||||||
|
xt_set
|
||||||
|
|
||||||
|
# nft extras (reject, fib — used by kube-proxy nf_tables rules)
|
||||||
|
nft_reject
|
||||||
|
nft_reject_ipv4
|
||||||
|
nft_reject_ipv6
|
||||||
|
nft_fib
|
||||||
|
nft_fib_ipv4
|
||||||
|
nft_fib_ipv6
|
||||||
|
|
||||||
|
# Reject targets (used by kube-proxy iptables-restore rules)
|
||||||
|
nf_reject_ipv4
|
||||||
|
nf_reject_ipv6
|
||||||
|
ipt_REJECT
|
||||||
|
ip6t_REJECT
|
||||||
|
|
||||||
|
# nfacct extension (kube-proxy probes for it)
|
||||||
|
xt_nfacct
|
||||||
|
|
||||||
|
# Networking — bridge and netfilter (required for K8s pod networking)
|
||||||
|
# Load order: llc → stp → bridge → br_netfilter
|
||||||
|
llc
|
||||||
|
stp
|
||||||
|
bridge
|
||||||
|
br_netfilter
|
||||||
|
veth
|
||||||
|
vxlan
|
||||||
|
|
||||||
|
# IPVS — useful for kube-proxy IPVS mode and CNI plugins
|
||||||
|
ip_vs
|
||||||
|
ip_vs_rr
|
||||||
|
ip_vs_wrr
|
||||||
|
ip_vs_sh
|
||||||
@@ -1,30 +1,78 @@
|
|||||||
# Kernel modules loaded at boot by init
|
# Kernel modules loaded at boot by init
|
||||||
# One module per line. Lines starting with # are ignored.
|
# One module per line. Lines starting with # are ignored.
|
||||||
# Modules are loaded in order listed.
|
# Modules are loaded in order listed — dependencies must come first.
|
||||||
|
|
||||||
# Networking — bridge and netfilter (required for K8s pod networking)
|
# Network device drivers (loaded early so interfaces are available)
|
||||||
br_netfilter
|
e1000
|
||||||
bridge
|
e1000e
|
||||||
veth
|
virtio_net
|
||||||
vxlan
|
|
||||||
|
|
||||||
# Netfilter / iptables (required for kube-proxy and service routing)
|
# Virtio support (for VMs — block, entropy)
|
||||||
ip_tables
|
virtio_blk
|
||||||
iptable_nat
|
virtio_rng
|
||||||
iptable_filter
|
|
||||||
iptable_mangle
|
|
||||||
nf_nat
|
|
||||||
nf_conntrack
|
|
||||||
nf_conntrack_netlink
|
|
||||||
|
|
||||||
# Filesystem — overlay (required for containerd)
|
# Filesystem — overlay (required for containerd)
|
||||||
overlay
|
overlay
|
||||||
|
|
||||||
# Conntrack (required for K8s services)
|
# Netfilter dependencies (must load before conntrack)
|
||||||
nf_conntrack
|
nf_defrag_ipv4
|
||||||
|
nf_defrag_ipv6
|
||||||
|
|
||||||
# Optional — useful for CNI plugins and diagnostics
|
# Netfilter / connection tracking (required for kube-proxy)
|
||||||
tun
|
nf_conntrack
|
||||||
|
nf_nat
|
||||||
|
nf_conntrack_netlink
|
||||||
|
|
||||||
|
# nftables (modern iptables backend — kernel 6.18 uses nf_tables, not ip_tables)
|
||||||
|
nf_tables
|
||||||
|
nft_compat
|
||||||
|
nft_chain_nat
|
||||||
|
nft_ct
|
||||||
|
nft_masq
|
||||||
|
nft_nat
|
||||||
|
nft_redir
|
||||||
|
|
||||||
|
# Netfilter xt match/target modules (used by kube-proxy iptables rules via nft_compat)
|
||||||
|
xt_conntrack
|
||||||
|
xt_MASQUERADE
|
||||||
|
xt_mark
|
||||||
|
xt_comment
|
||||||
|
xt_multiport
|
||||||
|
xt_nat
|
||||||
|
xt_addrtype
|
||||||
|
xt_connmark
|
||||||
|
xt_REDIRECT
|
||||||
|
xt_recent
|
||||||
|
xt_statistic
|
||||||
|
xt_set
|
||||||
|
|
||||||
|
# nft extras (reject, fib — used by kube-proxy nf_tables rules)
|
||||||
|
nft_reject
|
||||||
|
nft_reject_ipv4
|
||||||
|
nft_reject_ipv6
|
||||||
|
nft_fib
|
||||||
|
nft_fib_ipv4
|
||||||
|
nft_fib_ipv6
|
||||||
|
|
||||||
|
# Reject targets (used by kube-proxy iptables-restore rules)
|
||||||
|
nf_reject_ipv4
|
||||||
|
nf_reject_ipv6
|
||||||
|
ipt_REJECT
|
||||||
|
ip6t_REJECT
|
||||||
|
|
||||||
|
# nfacct extension (kube-proxy probes for it)
|
||||||
|
xt_nfacct
|
||||||
|
|
||||||
|
# Networking — bridge and netfilter (required for K8s pod networking)
|
||||||
|
# Load order: llc → stp → bridge → br_netfilter
|
||||||
|
llc
|
||||||
|
stp
|
||||||
|
bridge
|
||||||
|
br_netfilter
|
||||||
|
veth
|
||||||
|
vxlan
|
||||||
|
|
||||||
|
# IPVS — useful for kube-proxy IPVS mode and CNI plugins
|
||||||
ip_vs
|
ip_vs
|
||||||
ip_vs_rr
|
ip_vs_rr
|
||||||
ip_vs_wrr
|
ip_vs_wrr
|
||||||
|
|||||||
69
build/config/rpi-kernel-config.fragment
Normal file
69
build/config/rpi-kernel-config.fragment
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
# KubeSolo OS — Raspberry Pi kernel config overrides
|
||||||
|
# Applied on top of bcm2711_defconfig (Pi 4) or bcm2712_defconfig (Pi 5)
|
||||||
|
# These ensure container runtime support is enabled.
|
||||||
|
|
||||||
|
# cgroup v2 (mandatory for containerd/runc)
|
||||||
|
CONFIG_CGROUPS=y
|
||||||
|
CONFIG_CGROUP_CPUACCT=y
|
||||||
|
CONFIG_CGROUP_DEVICE=y
|
||||||
|
CONFIG_CGROUP_FREEZER=y
|
||||||
|
CONFIG_CGROUP_SCHED=y
|
||||||
|
CONFIG_CGROUP_PIDS=y
|
||||||
|
CONFIG_MEMCG=y
|
||||||
|
CONFIG_CGROUP_BPF=y
|
||||||
|
CONFIG_CFS_BANDWIDTH=y
|
||||||
|
|
||||||
|
# BPF (required for cgroup v2 device control)
|
||||||
|
CONFIG_BPF=y
|
||||||
|
CONFIG_BPF_SYSCALL=y
|
||||||
|
|
||||||
|
# Namespaces (mandatory for containers)
|
||||||
|
CONFIG_NAMESPACES=y
|
||||||
|
CONFIG_NET_NS=y
|
||||||
|
CONFIG_PID_NS=y
|
||||||
|
CONFIG_USER_NS=y
|
||||||
|
CONFIG_UTS_NS=y
|
||||||
|
CONFIG_IPC_NS=y
|
||||||
|
|
||||||
|
# Device management
|
||||||
|
CONFIG_DEVTMPFS=y
|
||||||
|
CONFIG_DEVTMPFS_MOUNT=y
|
||||||
|
|
||||||
|
# Filesystem
|
||||||
|
CONFIG_OVERLAY_FS=y
|
||||||
|
CONFIG_SQUASHFS=y
|
||||||
|
CONFIG_EXT4_FS=y
|
||||||
|
CONFIG_VFAT_FS=y
|
||||||
|
|
||||||
|
# Networking
|
||||||
|
CONFIG_BRIDGE=m
|
||||||
|
CONFIG_NETFILTER=y
|
||||||
|
CONFIG_NF_CONNTRACK=m
|
||||||
|
CONFIG_NF_NAT=m
|
||||||
|
CONFIG_NF_TABLES=m
|
||||||
|
CONFIG_VETH=m
|
||||||
|
CONFIG_VXLAN=m
|
||||||
|
|
||||||
|
# Security: AppArmor + Audit
|
||||||
|
CONFIG_AUDIT=y
|
||||||
|
CONFIG_AUDITSYSCALL=y
|
||||||
|
CONFIG_SECURITY=y
|
||||||
|
CONFIG_SECURITYFS=y
|
||||||
|
CONFIG_SECURITY_NETWORK=y
|
||||||
|
CONFIG_SECURITY_APPARMOR=y
|
||||||
|
CONFIG_DEFAULT_SECURITY_APPARMOR=y
|
||||||
|
|
||||||
|
# Security: seccomp
|
||||||
|
CONFIG_SECCOMP=y
|
||||||
|
CONFIG_SECCOMP_FILTER=y
|
||||||
|
|
||||||
|
# Crypto (image verification)
|
||||||
|
CONFIG_CRYPTO_SHA256=y
|
||||||
|
|
||||||
|
# Disable unnecessary subsystems for edge appliance
|
||||||
|
# CONFIG_SOUND is not set
|
||||||
|
# CONFIG_DRM is not set
|
||||||
|
# CONFIG_MEDIA_SUPPORT is not set
|
||||||
|
# CONFIG_WIRELESS is not set
|
||||||
|
# CONFIG_BT is not set
|
||||||
|
# CONFIG_NFC is not set
|
||||||
@@ -15,5 +15,28 @@ KUBESOLO_INSTALL_URL=https://get.kubesolo.io
|
|||||||
GRUB_VERSION=2.12
|
GRUB_VERSION=2.12
|
||||||
SYSLINUX_VERSION=6.03
|
SYSLINUX_VERSION=6.03
|
||||||
|
|
||||||
|
# SHA256 checksums for supply chain verification
|
||||||
|
# Populate by running: sha256sum build/cache/<file>
|
||||||
|
# Leave empty to skip verification (useful for first fetch)
|
||||||
|
TINYCORE_ISO_SHA256=""
|
||||||
|
KUBESOLO_SHA256=""
|
||||||
|
NETFILTER_TCZ_SHA256=""
|
||||||
|
NET_BRIDGING_TCZ_SHA256=""
|
||||||
|
IPTABLES_TCZ_SHA256=""
|
||||||
|
|
||||||
|
# piCore64 (ARM64 — Raspberry Pi)
|
||||||
|
PICORE_VERSION=15.0.0
|
||||||
|
PICORE_ARCH=aarch64
|
||||||
|
PICORE_IMAGE=piCore64-${PICORE_VERSION}.zip
|
||||||
|
PICORE_IMAGE_URL=http://www.tinycorelinux.net/${PICORE_VERSION%%.*}.x/${PICORE_ARCH}/releases/RPi/${PICORE_IMAGE}
|
||||||
|
|
||||||
|
# Raspberry Pi firmware (boot blobs, DTBs)
|
||||||
|
RPI_FIRMWARE_TAG=1.20240529
|
||||||
|
RPI_FIRMWARE_URL=https://github.com/raspberrypi/firmware/archive/refs/tags/${RPI_FIRMWARE_TAG}.tar.gz
|
||||||
|
|
||||||
|
# Raspberry Pi kernel source
|
||||||
|
RPI_KERNEL_BRANCH=rpi-6.6.y
|
||||||
|
RPI_KERNEL_REPO=https://github.com/raspberrypi/linux
|
||||||
|
|
||||||
# Output naming
|
# Output naming
|
||||||
OS_NAME=kubesolo-os
|
OS_NAME=kubesolo-os
|
||||||
|
|||||||
52
build/rootfs/etc/apparmor.d/containerd
Normal file
52
build/rootfs/etc/apparmor.d/containerd
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
# AppArmor profile for containerd
|
||||||
|
# Start in complain mode to log without blocking
|
||||||
|
|
||||||
|
#include <tunables/global>
|
||||||
|
|
||||||
|
profile containerd /usr/bin/containerd flags=(complain) {
|
||||||
|
#include <abstractions/base>
|
||||||
|
|
||||||
|
# Binary and shared libraries
|
||||||
|
/usr/bin/containerd mr,
|
||||||
|
/usr/lib/** mr,
|
||||||
|
/lib/** mr,
|
||||||
|
|
||||||
|
# Containerd runtime state
|
||||||
|
/var/lib/containerd/** rw,
|
||||||
|
/run/containerd/** rw,
|
||||||
|
|
||||||
|
# Container image layers and snapshots
|
||||||
|
/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/** rw,
|
||||||
|
|
||||||
|
# CNI networking
|
||||||
|
/etc/cni/** r,
|
||||||
|
/opt/cni/bin/** ix,
|
||||||
|
|
||||||
|
# Proc and sys access for containers
|
||||||
|
@{PROC}/** r,
|
||||||
|
/sys/** r,
|
||||||
|
|
||||||
|
# Device access for containers
|
||||||
|
/dev/** rw,
|
||||||
|
|
||||||
|
# Network access
|
||||||
|
network,
|
||||||
|
|
||||||
|
# Container runtime needs broad capabilities
|
||||||
|
capability,
|
||||||
|
|
||||||
|
# Allow executing container runtimes
|
||||||
|
/usr/bin/containerd-shim-runc-v2 ix,
|
||||||
|
/usr/bin/runc ix,
|
||||||
|
/usr/sbin/runc ix,
|
||||||
|
|
||||||
|
# Temp files
|
||||||
|
/tmp/** rw,
|
||||||
|
|
||||||
|
# Log files
|
||||||
|
/var/log/** rw,
|
||||||
|
|
||||||
|
# Signal handling for child processes
|
||||||
|
signal,
|
||||||
|
ptrace,
|
||||||
|
}
|
||||||
56
build/rootfs/etc/apparmor.d/kubelet
Normal file
56
build/rootfs/etc/apparmor.d/kubelet
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
# AppArmor profile for kubesolo (kubelet + control plane)
|
||||||
|
# Start in complain mode to log without blocking
|
||||||
|
|
||||||
|
#include <tunables/global>
|
||||||
|
|
||||||
|
profile kubesolo /usr/bin/kubesolo flags=(complain) {
|
||||||
|
#include <abstractions/base>
|
||||||
|
|
||||||
|
# Binary and shared libraries
|
||||||
|
/usr/bin/kubesolo mr,
|
||||||
|
/usr/lib/** mr,
|
||||||
|
/lib/** mr,
|
||||||
|
|
||||||
|
# KubeSolo state (etcd/SQLite, certificates, manifests)
|
||||||
|
/var/lib/kubesolo/** rw,
|
||||||
|
|
||||||
|
# KubeSolo configuration
|
||||||
|
/etc/kubesolo/** r,
|
||||||
|
|
||||||
|
# Containerd socket
|
||||||
|
/run/containerd/** rw,
|
||||||
|
|
||||||
|
# CNI networking
|
||||||
|
/etc/cni/** r,
|
||||||
|
/opt/cni/bin/** ix,
|
||||||
|
|
||||||
|
# Proc and sys access
|
||||||
|
@{PROC}/** r,
|
||||||
|
/sys/** r,
|
||||||
|
|
||||||
|
# Device access
|
||||||
|
/dev/** rw,
|
||||||
|
|
||||||
|
# Network access (API server, kubelet, etcd)
|
||||||
|
network,
|
||||||
|
|
||||||
|
# Control plane needs broad capabilities
|
||||||
|
capability,
|
||||||
|
|
||||||
|
# Kubectl and other tools
|
||||||
|
/usr/bin/kubectl ix,
|
||||||
|
/usr/local/bin/** ix,
|
||||||
|
|
||||||
|
# Temp files
|
||||||
|
/tmp/** rw,
|
||||||
|
|
||||||
|
# Log files
|
||||||
|
/var/log/** rw,
|
||||||
|
|
||||||
|
# Kubelet needs to manage pods
|
||||||
|
/var/lib/kubelet/** rw,
|
||||||
|
|
||||||
|
# Signal handling
|
||||||
|
signal,
|
||||||
|
ptrace,
|
||||||
|
}
|
||||||
27
build/rootfs/etc/sysctl.d/security.conf
Normal file
27
build/rootfs/etc/sysctl.d/security.conf
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Security hardening — applied automatically by 40-sysctl.sh
|
||||||
|
# Network: anti-spoofing
|
||||||
|
net.ipv4.conf.all.rp_filter = 1
|
||||||
|
net.ipv4.conf.default.rp_filter = 1
|
||||||
|
# Network: SYN flood protection
|
||||||
|
net.ipv4.tcp_syncookies = 1
|
||||||
|
# Network: ICMP hardening
|
||||||
|
net.ipv4.conf.all.accept_redirects = 0
|
||||||
|
net.ipv4.conf.default.accept_redirects = 0
|
||||||
|
net.ipv4.conf.all.send_redirects = 0
|
||||||
|
net.ipv4.conf.default.send_redirects = 0
|
||||||
|
net.ipv4.icmp_echo_ignore_broadcasts = 1
|
||||||
|
net.ipv4.icmp_ignore_bogus_error_responses = 1
|
||||||
|
net.ipv4.conf.all.log_martians = 1
|
||||||
|
# Network: IPv6 hardening
|
||||||
|
net.ipv6.conf.all.accept_redirects = 0
|
||||||
|
net.ipv6.conf.default.accept_redirects = 0
|
||||||
|
net.ipv6.conf.all.accept_ra = 0
|
||||||
|
# Network: source routing
|
||||||
|
net.ipv4.conf.all.accept_source_route = 0
|
||||||
|
net.ipv4.conf.default.accept_source_route = 0
|
||||||
|
# Kernel: information disclosure
|
||||||
|
kernel.kptr_restrict = 2
|
||||||
|
kernel.dmesg_restrict = 1
|
||||||
|
kernel.perf_event_paranoid = 3
|
||||||
|
# Kernel: core dump safety
|
||||||
|
fs.suid_dumpable = 0
|
||||||
158
build/scripts/build-kernel-arm64.sh
Executable file
158
build/scripts/build-kernel-arm64.sh
Executable file
@@ -0,0 +1,158 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# build-kernel-arm64.sh — Build ARM64 kernel for Raspberry Pi 4/5
|
||||||
|
#
|
||||||
|
# Uses the official raspberrypi/linux kernel fork with bcm2711_defconfig
|
||||||
|
# as the base, overlaid with container-critical config options.
|
||||||
|
#
|
||||||
|
# Output is cached in $CACHE_DIR/custom-kernel-arm64/ and reused across builds.
|
||||||
|
#
|
||||||
|
# Requirements:
|
||||||
|
# - gcc-aarch64-linux-gnu (cross-compiler)
|
||||||
|
# - Standard kernel build deps (bc, bison, flex, etc.)
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
||||||
|
|
||||||
|
# shellcheck source=../config/versions.env
|
||||||
|
. "$SCRIPT_DIR/../config/versions.env"
|
||||||
|
|
||||||
|
CUSTOM_KERNEL_DIR="$CACHE_DIR/custom-kernel-arm64"
|
||||||
|
CUSTOM_IMAGE="$CUSTOM_KERNEL_DIR/Image"
|
||||||
|
CUSTOM_MODULES="$CUSTOM_KERNEL_DIR/modules"
|
||||||
|
CUSTOM_DTBS="$CUSTOM_KERNEL_DIR/dtbs"
|
||||||
|
|
||||||
|
mkdir -p "$CACHE_DIR" "$CUSTOM_KERNEL_DIR"
|
||||||
|
|
||||||
|
# --- Skip if already built ---
|
||||||
|
if [ -f "$CUSTOM_IMAGE" ] && [ -d "$CUSTOM_MODULES" ]; then
|
||||||
|
echo "==> ARM64 kernel already built (cached)"
|
||||||
|
echo " Image: $CUSTOM_IMAGE ($(du -h "$CUSTOM_IMAGE" | cut -f1))"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Verify cross-compiler ---
|
||||||
|
if ! command -v aarch64-linux-gnu-gcc >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: aarch64-linux-gnu-gcc not found"
|
||||||
|
echo "Install: apt-get install gcc-aarch64-linux-gnu"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Building ARM64 kernel for Raspberry Pi..."
|
||||||
|
echo " Branch: $RPI_KERNEL_BRANCH"
|
||||||
|
echo " Repo: $RPI_KERNEL_REPO"
|
||||||
|
|
||||||
|
# --- Download kernel source ---
|
||||||
|
KERNEL_SRC_DIR="$CACHE_DIR/rpi-linux-${RPI_KERNEL_BRANCH}"
|
||||||
|
if [ ! -d "$KERNEL_SRC_DIR" ]; then
|
||||||
|
echo "==> Downloading RPi kernel source (shallow clone)..."
|
||||||
|
git clone --depth 1 --branch "$RPI_KERNEL_BRANCH" \
|
||||||
|
"$RPI_KERNEL_REPO" "$KERNEL_SRC_DIR"
|
||||||
|
else
|
||||||
|
echo "==> Kernel source already cached"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Build in /tmp for case-sensitivity ---
|
||||||
|
KERNEL_BUILD_DIR="/tmp/kernel-build-arm64"
|
||||||
|
rm -rf "$KERNEL_BUILD_DIR"
|
||||||
|
cp -a "$KERNEL_SRC_DIR" "$KERNEL_BUILD_DIR"
|
||||||
|
|
||||||
|
cd "$KERNEL_BUILD_DIR"
|
||||||
|
|
||||||
|
# --- Apply base config (Pi 4 = bcm2711) ---
|
||||||
|
echo "==> Applying bcm2711_defconfig..."
|
||||||
|
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- bcm2711_defconfig
|
||||||
|
|
||||||
|
# --- Apply container config overrides ---
|
||||||
|
CONFIG_FRAGMENT="$PROJECT_ROOT/build/config/rpi-kernel-config.fragment"
|
||||||
|
if [ -f "$CONFIG_FRAGMENT" ]; then
|
||||||
|
echo "==> Applying KubeSolo config overrides..."
|
||||||
|
while IFS= read -r line; do
|
||||||
|
# Skip comments and empty lines
|
||||||
|
case "$line" in \#*|"") continue ;; esac
|
||||||
|
key="${line%%=*}"
|
||||||
|
value="${line#*=}"
|
||||||
|
case "$value" in
|
||||||
|
y) ./scripts/config --enable "$key" ;;
|
||||||
|
m) ./scripts/config --module "$key" ;;
|
||||||
|
n) ./scripts/config --disable "${key#CONFIG_}" ;;
|
||||||
|
*) ./scripts/config --set-str "$key" "$value" ;;
|
||||||
|
esac
|
||||||
|
done < "$CONFIG_FRAGMENT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Handle "is not set" comments as disables
|
||||||
|
if [ -f "$CONFIG_FRAGMENT" ]; then
|
||||||
|
while IFS= read -r line; do
|
||||||
|
case "$line" in
|
||||||
|
"# CONFIG_"*" is not set")
|
||||||
|
key=$(echo "$line" | sed -n 's/^# \(CONFIG_[A-Z_]*\) is not set$/\1/p')
|
||||||
|
[ -n "$key" ] && ./scripts/config --disable "${key#CONFIG_}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done < "$CONFIG_FRAGMENT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Resolve dependencies
|
||||||
|
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- olddefconfig
|
||||||
|
|
||||||
|
# --- Build kernel + modules + DTBs ---
|
||||||
|
NPROC=$(nproc 2>/dev/null || echo 4)
|
||||||
|
echo ""
|
||||||
|
echo "==> Building ARM64 kernel (${NPROC} parallel jobs)..."
|
||||||
|
echo " This may take 20-30 minutes..."
|
||||||
|
|
||||||
|
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -j"$NPROC" Image modules dtbs 2>&1
|
||||||
|
|
||||||
|
echo "==> ARM64 kernel build complete"
|
||||||
|
|
||||||
|
# --- Install to staging ---
|
||||||
|
echo "==> Installing Image..."
|
||||||
|
cp arch/arm64/boot/Image "$CUSTOM_IMAGE"
|
||||||
|
|
||||||
|
echo "==> Installing modules (stripped)..."
|
||||||
|
rm -rf "$CUSTOM_MODULES"
|
||||||
|
mkdir -p "$CUSTOM_MODULES"
|
||||||
|
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- \
|
||||||
|
INSTALL_MOD_STRIP=1 modules_install INSTALL_MOD_PATH="$CUSTOM_MODULES"
|
||||||
|
|
||||||
|
# Remove build/source symlinks
|
||||||
|
KVER=$(ls "$CUSTOM_MODULES/lib/modules/" | head -1)
|
||||||
|
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/build"
|
||||||
|
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/source"
|
||||||
|
|
||||||
|
# Run depmod
|
||||||
|
depmod -a -b "$CUSTOM_MODULES" "$KVER" 2>/dev/null || true
|
||||||
|
|
||||||
|
echo "==> Installing Device Tree Blobs..."
|
||||||
|
rm -rf "$CUSTOM_DTBS"
|
||||||
|
mkdir -p "$CUSTOM_DTBS/overlays"
|
||||||
|
# Pi 4 DTBs
|
||||||
|
cp arch/arm64/boot/dts/broadcom/bcm2711*.dtb "$CUSTOM_DTBS/" 2>/dev/null || true
|
||||||
|
# Pi 5 DTBs
|
||||||
|
cp arch/arm64/boot/dts/broadcom/bcm2712*.dtb "$CUSTOM_DTBS/" 2>/dev/null || true
|
||||||
|
# Overlays we need
|
||||||
|
for overlay in disable-wifi disable-bt; do
|
||||||
|
[ -f "arch/arm64/boot/dts/overlays/${overlay}.dtbo" ] && \
|
||||||
|
cp "arch/arm64/boot/dts/overlays/${overlay}.dtbo" "$CUSTOM_DTBS/overlays/"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Save config for reference
|
||||||
|
cp .config "$CUSTOM_KERNEL_DIR/.config"
|
||||||
|
|
||||||
|
# --- Clean up ---
|
||||||
|
echo "==> Cleaning kernel build directory..."
|
||||||
|
cd /
|
||||||
|
rm -rf "$KERNEL_BUILD_DIR"
|
||||||
|
|
||||||
|
# --- Summary ---
|
||||||
|
echo ""
|
||||||
|
echo "==> ARM64 kernel build complete:"
|
||||||
|
echo " Image: $CUSTOM_IMAGE ($(du -h "$CUSTOM_IMAGE" | cut -f1))"
|
||||||
|
echo " Kernel ver: $KVER"
|
||||||
|
MOD_COUNT=$(find "$CUSTOM_MODULES/lib/modules/$KVER" -name '*.ko*' 2>/dev/null | wc -l)
|
||||||
|
echo " Modules: $MOD_COUNT"
|
||||||
|
echo " Modules size: $(du -sh "$CUSTOM_MODULES/lib/modules/$KVER" 2>/dev/null | cut -f1)"
|
||||||
|
echo " DTBs: $(ls "$CUSTOM_DTBS"/*.dtb 2>/dev/null | wc -l)"
|
||||||
|
echo ""
|
||||||
240
build/scripts/build-kernel.sh
Executable file
240
build/scripts/build-kernel.sh
Executable file
@@ -0,0 +1,240 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# build-kernel.sh — Build custom Tiny Core kernel with CONFIG_CGROUP_BPF=y
|
||||||
|
#
|
||||||
|
# The stock Tiny Core 17.0 kernel (6.18.2-tinycore64) lacks CONFIG_CGROUP_BPF,
|
||||||
|
# which is required for cgroup v2 device control in runc/containerd.
|
||||||
|
# This script downloads the TC-patched kernel source, enables CONFIG_CGROUP_BPF,
|
||||||
|
# and builds vmlinuz + modules.
|
||||||
|
#
|
||||||
|
# Output is cached in $CACHE_DIR/custom-kernel/ and reused across builds.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
||||||
|
|
||||||
|
# shellcheck source=../config/versions.env
|
||||||
|
. "$SCRIPT_DIR/../config/versions.env"
|
||||||
|
|
||||||
|
KVER="6.18.2-tinycore64"
|
||||||
|
KERNEL_BASE_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/release/src/kernel"
|
||||||
|
KERNEL_SRC_URL="${KERNEL_BASE_URL}/linux-6.18.2-patched.tar.xz"
|
||||||
|
KERNEL_CFG_URL="${KERNEL_BASE_URL}/config-${KVER}"
|
||||||
|
|
||||||
|
CUSTOM_KERNEL_DIR="$CACHE_DIR/custom-kernel"
|
||||||
|
CUSTOM_VMLINUZ="$CUSTOM_KERNEL_DIR/vmlinuz"
|
||||||
|
CUSTOM_MODULES="$CUSTOM_KERNEL_DIR/modules"
|
||||||
|
|
||||||
|
mkdir -p "$CACHE_DIR" "$CUSTOM_KERNEL_DIR"
|
||||||
|
|
||||||
|
# --- Skip if already built ---
|
||||||
|
if [ -f "$CUSTOM_VMLINUZ" ] && [ -d "$CUSTOM_MODULES/lib/modules/$KVER" ]; then
|
||||||
|
echo "==> Custom kernel already built (cached)"
|
||||||
|
echo " vmlinuz: $CUSTOM_VMLINUZ ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||||
|
MOD_COUNT=$(find "$CUSTOM_MODULES/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||||
|
echo " Modules: $MOD_COUNT modules in $CUSTOM_MODULES/lib/modules/$KVER"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Building custom kernel with CONFIG_CGROUP_BPF=y..."
|
||||||
|
echo " Kernel version: $KVER"
|
||||||
|
|
||||||
|
# --- Download kernel source ---
|
||||||
|
KERNEL_SRC_ARCHIVE="$CACHE_DIR/linux-6.18.2-patched.tar.xz"
|
||||||
|
if [ ! -f "$KERNEL_SRC_ARCHIVE" ]; then
|
||||||
|
echo "==> Downloading kernel source (~149 MB)..."
|
||||||
|
echo " URL: $KERNEL_SRC_URL"
|
||||||
|
wget -q --show-progress -O "$KERNEL_SRC_ARCHIVE" "$KERNEL_SRC_URL" 2>/dev/null || \
|
||||||
|
curl -fSL "$KERNEL_SRC_URL" -o "$KERNEL_SRC_ARCHIVE"
|
||||||
|
echo " Downloaded: $(du -h "$KERNEL_SRC_ARCHIVE" | cut -f1)"
|
||||||
|
else
|
||||||
|
echo "==> Kernel source already cached: $(du -h "$KERNEL_SRC_ARCHIVE" | cut -f1)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Download stock config ---
|
||||||
|
KERNEL_CFG="$CACHE_DIR/config-${KVER}"
|
||||||
|
if [ ! -f "$KERNEL_CFG" ]; then
|
||||||
|
echo "==> Downloading stock kernel config..."
|
||||||
|
echo " URL: $KERNEL_CFG_URL"
|
||||||
|
wget -q -O "$KERNEL_CFG" "$KERNEL_CFG_URL" 2>/dev/null || \
|
||||||
|
curl -fSL "$KERNEL_CFG_URL" -o "$KERNEL_CFG"
|
||||||
|
else
|
||||||
|
echo "==> Stock kernel config already cached"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Extract source ---
|
||||||
|
# IMPORTANT: Must extract on a case-sensitive filesystem. The kernel source has
|
||||||
|
# files that differ only by case (e.g., xt_mark.h vs xt_MARK.h). If the cache
|
||||||
|
# is on macOS (case-insensitive APFS), extraction silently loses files.
|
||||||
|
# Use /tmp inside the container (ext4, case-sensitive) for the build.
|
||||||
|
KERNEL_BUILD_DIR="/tmp/kernel-build"
|
||||||
|
rm -rf "$KERNEL_BUILD_DIR"
|
||||||
|
mkdir -p "$KERNEL_BUILD_DIR"
|
||||||
|
|
||||||
|
echo "==> Extracting kernel source (case-sensitive filesystem)..."
|
||||||
|
tar -xf "$KERNEL_SRC_ARCHIVE" -C "$KERNEL_BUILD_DIR"
|
||||||
|
|
||||||
|
# Find the extracted source directory (could be linux-6.18.2 or linux-6.18.2-patched)
|
||||||
|
KERNEL_SRC_DIR=$(find "$KERNEL_BUILD_DIR" -maxdepth 1 -type d -name 'linux-*' | head -1)
|
||||||
|
if [ -z "$KERNEL_SRC_DIR" ]; then
|
||||||
|
echo "ERROR: Could not find kernel source directory after extraction"
|
||||||
|
ls -la "$KERNEL_BUILD_DIR"/
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo " Source dir: $(basename "$KERNEL_SRC_DIR")"
|
||||||
|
|
||||||
|
cd "$KERNEL_SRC_DIR"
|
||||||
|
|
||||||
|
# --- Apply stock config + enable CONFIG_CGROUP_BPF ---
|
||||||
|
echo "==> Applying stock Tiny Core config..."
|
||||||
|
cp "$KERNEL_CFG" .config
|
||||||
|
|
||||||
|
echo "==> Enabling required kernel configs..."
|
||||||
|
./scripts/config --enable CONFIG_CGROUP_BPF
|
||||||
|
./scripts/config --enable CONFIG_DEVTMPFS
|
||||||
|
./scripts/config --enable CONFIG_DEVTMPFS_MOUNT
|
||||||
|
./scripts/config --enable CONFIG_MEMCG
|
||||||
|
./scripts/config --enable CONFIG_CFS_BANDWIDTH
|
||||||
|
|
||||||
|
# --- Strip unnecessary subsystems for smallest footprint ---
|
||||||
|
# This is a headless K8s edge appliance — no sound, GPU, wireless, etc.
|
||||||
|
echo "==> Disabling unnecessary subsystems for minimal footprint..."
|
||||||
|
|
||||||
|
# Sound subsystem (not needed on headless appliance)
|
||||||
|
./scripts/config --disable SOUND
|
||||||
|
|
||||||
|
# GPU/DRM (serial console only, no display)
|
||||||
|
./scripts/config --disable DRM
|
||||||
|
|
||||||
|
# KVM hypervisor (this IS the guest/bare metal, not a hypervisor)
|
||||||
|
./scripts/config --disable KVM
|
||||||
|
|
||||||
|
# Media/camera/TV/radio (not needed)
|
||||||
|
./scripts/config --disable MEDIA_SUPPORT
|
||||||
|
|
||||||
|
# Wireless networking (wired edge device)
|
||||||
|
./scripts/config --disable WIRELESS
|
||||||
|
./scripts/config --disable WLAN
|
||||||
|
./scripts/config --disable CFG80211
|
||||||
|
|
||||||
|
# Bluetooth (not needed)
|
||||||
|
./scripts/config --disable BT
|
||||||
|
|
||||||
|
# NFC (not needed)
|
||||||
|
./scripts/config --disable NFC
|
||||||
|
|
||||||
|
# Infiniband (not needed on edge)
|
||||||
|
./scripts/config --disable INFINIBAND
|
||||||
|
|
||||||
|
# PCMCIA (legacy, not needed)
|
||||||
|
./scripts/config --disable PCMCIA
|
||||||
|
|
||||||
|
# Amateur radio (not needed)
|
||||||
|
./scripts/config --disable HAMRADIO
|
||||||
|
|
||||||
|
# ISDN (not needed)
|
||||||
|
./scripts/config --disable ISDN
|
||||||
|
|
||||||
|
# ATM networking (not needed)
|
||||||
|
./scripts/config --disable ATM
|
||||||
|
|
||||||
|
# Joystick/gamepad (not needed)
|
||||||
|
./scripts/config --disable INPUT_JOYSTICK
|
||||||
|
./scripts/config --disable INPUT_TABLET
|
||||||
|
|
||||||
|
# FPGA (not needed)
|
||||||
|
./scripts/config --disable FPGA
|
||||||
|
|
||||||
|
# First pass: resolve base dependencies before adding security configs.
|
||||||
|
# The stock TC config has "# CONFIG_SECURITY is not set" which causes
|
||||||
|
# olddefconfig to strip security-related options if applied in a single pass.
|
||||||
|
make olddefconfig
|
||||||
|
|
||||||
|
# Security: AppArmor LSM + Audit subsystem
|
||||||
|
# Applied AFTER first olddefconfig to ensure CONFIG_SECURITY dependencies
|
||||||
|
# (SYSFS, MULTIUSER) are resolved before enabling the security subtree.
|
||||||
|
echo "==> Enabling AppArmor + Audit kernel configs..."
|
||||||
|
./scripts/config --enable CONFIG_AUDIT
|
||||||
|
./scripts/config --enable CONFIG_AUDITSYSCALL
|
||||||
|
./scripts/config --enable CONFIG_SECURITY
|
||||||
|
./scripts/config --enable CONFIG_SECURITYFS
|
||||||
|
./scripts/config --enable CONFIG_SECURITY_NETWORK
|
||||||
|
./scripts/config --enable CONFIG_SECURITY_APPARMOR
|
||||||
|
./scripts/config --set-str CONFIG_LSM "lockdown,yama,apparmor"
|
||||||
|
./scripts/config --set-str CONFIG_DEFAULT_SECURITY "apparmor"
|
||||||
|
|
||||||
|
# Second pass: resolve security config dependencies
|
||||||
|
make olddefconfig
|
||||||
|
|
||||||
|
# Verify critical configs are set
|
||||||
|
if ! grep -q 'CONFIG_CGROUP_BPF=y' .config; then
|
||||||
|
echo "ERROR: CONFIG_CGROUP_BPF not set after olddefconfig"
|
||||||
|
grep 'CGROUP_BPF' .config || echo " (CGROUP_BPF not found in .config)"
|
||||||
|
echo ""
|
||||||
|
echo "Prerequisites check:"
|
||||||
|
grep -E 'CONFIG_BPF=|CONFIG_BPF_SYSCALL=' .config || echo " BPF not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo " CONFIG_CGROUP_BPF=y confirmed"
|
||||||
|
|
||||||
|
if ! grep -q 'CONFIG_SECURITY_APPARMOR=y' .config; then
|
||||||
|
echo "ERROR: CONFIG_SECURITY_APPARMOR not set after olddefconfig"
|
||||||
|
echo " Security-related configs:"
|
||||||
|
grep -E 'CONFIG_SECURITY=|CONFIG_SECURITYFS=|CONFIG_SECURITY_APPARMOR=' .config
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo " CONFIG_SECURITY_APPARMOR=y confirmed"
|
||||||
|
|
||||||
|
if ! grep -q 'CONFIG_AUDIT=y' .config; then
|
||||||
|
echo "ERROR: CONFIG_AUDIT not set after olddefconfig"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo " CONFIG_AUDIT=y confirmed"
|
||||||
|
|
||||||
|
# Show what changed (security-related)
|
||||||
|
echo " Key config values:"
|
||||||
|
grep -E 'CONFIG_SECURITY=|CONFIG_SECURITY_APPARMOR=|CONFIG_AUDIT=|CONFIG_LSM=|CONFIG_CGROUP_BPF=' .config | sed 's/^/ /'
|
||||||
|
|
||||||
|
# --- Build kernel + modules ---
|
||||||
|
NPROC=$(nproc 2>/dev/null || echo 4)
|
||||||
|
echo ""
|
||||||
|
echo "==> Building kernel (${NPROC} parallel jobs)..."
|
||||||
|
echo " This may take 15-25 minutes..."
|
||||||
|
|
||||||
|
make -j"$NPROC" bzImage modules 2>&1
|
||||||
|
|
||||||
|
echo "==> Kernel build complete"
|
||||||
|
|
||||||
|
# --- Install to staging ---
|
||||||
|
echo "==> Installing vmlinuz..."
|
||||||
|
cp arch/x86/boot/bzImage "$CUSTOM_VMLINUZ"
|
||||||
|
|
||||||
|
echo "==> Installing modules (stripped)..."
|
||||||
|
rm -rf "$CUSTOM_MODULES"
|
||||||
|
mkdir -p "$CUSTOM_MODULES"
|
||||||
|
make INSTALL_MOD_STRIP=1 modules_install INSTALL_MOD_PATH="$CUSTOM_MODULES"
|
||||||
|
|
||||||
|
# Remove build/source symlinks (they point to the build dir which won't exist in rootfs)
|
||||||
|
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/build"
|
||||||
|
rm -f "$CUSTOM_MODULES/lib/modules/$KVER/source"
|
||||||
|
|
||||||
|
# Run depmod to generate proper module dependency files
|
||||||
|
echo "==> Running depmod..."
|
||||||
|
depmod -a -b "$CUSTOM_MODULES" "$KVER" 2>/dev/null || true
|
||||||
|
|
||||||
|
# Save the final config for reference
|
||||||
|
cp .config "$CUSTOM_KERNEL_DIR/.config"
|
||||||
|
|
||||||
|
# --- Clean up build dir (large, ~1.5 GB) ---
|
||||||
|
echo "==> Cleaning kernel build directory..."
|
||||||
|
cd /
|
||||||
|
rm -rf "$KERNEL_BUILD_DIR"
|
||||||
|
|
||||||
|
# --- Summary ---
|
||||||
|
echo ""
|
||||||
|
echo "==> Custom kernel build complete:"
|
||||||
|
echo " vmlinuz: $CUSTOM_VMLINUZ ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||||
|
MOD_COUNT=$(find "$CUSTOM_MODULES/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||||
|
echo " Modules: $MOD_COUNT modules"
|
||||||
|
echo " Modules size: $(du -sh "$CUSTOM_MODULES/lib/modules/$KVER" | cut -f1)"
|
||||||
|
echo ""
|
||||||
@@ -51,10 +51,39 @@ size=1048576, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, name="SystemB"
|
|||||||
type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, name="Data"
|
type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, name="Data"
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Set up loop device
|
# Set up loop device with partition mappings
|
||||||
LOOP=$(losetup --show -fP "$IMG_OUTPUT")
|
LOOP=$(losetup --show -f "$IMG_OUTPUT")
|
||||||
echo "==> Loop device: $LOOP"
|
echo "==> Loop device: $LOOP"
|
||||||
|
|
||||||
|
# Use kpartx for reliable partition device nodes (works in Docker/containers)
|
||||||
|
USE_KPARTX=false
|
||||||
|
if [ ! -b "${LOOP}p1" ]; then
|
||||||
|
if command -v kpartx >/dev/null 2>&1; then
|
||||||
|
kpartx -a "$LOOP"
|
||||||
|
USE_KPARTX=true
|
||||||
|
sleep 1
|
||||||
|
LOOP_NAME=$(basename "$LOOP")
|
||||||
|
P1="/dev/mapper/${LOOP_NAME}p1"
|
||||||
|
P2="/dev/mapper/${LOOP_NAME}p2"
|
||||||
|
P3="/dev/mapper/${LOOP_NAME}p3"
|
||||||
|
P4="/dev/mapper/${LOOP_NAME}p4"
|
||||||
|
else
|
||||||
|
# Retry with -P flag
|
||||||
|
losetup -d "$LOOP"
|
||||||
|
LOOP=$(losetup --show -fP "$IMG_OUTPUT")
|
||||||
|
sleep 1
|
||||||
|
P1="${LOOP}p1"
|
||||||
|
P2="${LOOP}p2"
|
||||||
|
P3="${LOOP}p3"
|
||||||
|
P4="${LOOP}p4"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
P1="${LOOP}p1"
|
||||||
|
P2="${LOOP}p2"
|
||||||
|
P3="${LOOP}p3"
|
||||||
|
P4="${LOOP}p4"
|
||||||
|
fi
|
||||||
|
|
||||||
MNT_EFI=$(mktemp -d)
|
MNT_EFI=$(mktemp -d)
|
||||||
MNT_SYSA=$(mktemp -d)
|
MNT_SYSA=$(mktemp -d)
|
||||||
MNT_SYSB=$(mktemp -d)
|
MNT_SYSB=$(mktemp -d)
|
||||||
@@ -65,22 +94,25 @@ cleanup() {
|
|||||||
umount "$MNT_SYSA" 2>/dev/null || true
|
umount "$MNT_SYSA" 2>/dev/null || true
|
||||||
umount "$MNT_SYSB" 2>/dev/null || true
|
umount "$MNT_SYSB" 2>/dev/null || true
|
||||||
umount "$MNT_DATA" 2>/dev/null || true
|
umount "$MNT_DATA" 2>/dev/null || true
|
||||||
|
if [ "$USE_KPARTX" = true ]; then
|
||||||
|
kpartx -d "$LOOP" 2>/dev/null || true
|
||||||
|
fi
|
||||||
losetup -d "$LOOP" 2>/dev/null || true
|
losetup -d "$LOOP" 2>/dev/null || true
|
||||||
rm -rf "$MNT_EFI" "$MNT_SYSA" "$MNT_SYSB" "$MNT_DATA" 2>/dev/null || true
|
rm -rf "$MNT_EFI" "$MNT_SYSA" "$MNT_SYSB" "$MNT_DATA" 2>/dev/null || true
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
# Format partitions
|
# Format partitions
|
||||||
mkfs.vfat -F 32 -n KSOLOEFI "${LOOP}p1"
|
mkfs.vfat -F 32 -n KSOLOEFI "$P1"
|
||||||
mkfs.ext4 -q -L KSOLOA "${LOOP}p2"
|
mkfs.ext4 -q -L KSOLOA "$P2"
|
||||||
mkfs.ext4 -q -L KSOLOB "${LOOP}p3"
|
mkfs.ext4 -q -L KSOLOB "$P3"
|
||||||
mkfs.ext4 -q -L KSOLODATA "${LOOP}p4"
|
mkfs.ext4 -q -L KSOLODATA "$P4"
|
||||||
|
|
||||||
# Mount all partitions
|
# Mount all partitions
|
||||||
mount "${LOOP}p1" "$MNT_EFI"
|
mount "$P1" "$MNT_EFI"
|
||||||
mount "${LOOP}p2" "$MNT_SYSA"
|
mount "$P2" "$MNT_SYSA"
|
||||||
mount "${LOOP}p3" "$MNT_SYSB"
|
mount "$P3" "$MNT_SYSB"
|
||||||
mount "${LOOP}p4" "$MNT_DATA"
|
mount "$P4" "$MNT_DATA"
|
||||||
|
|
||||||
# --- EFI/Boot Partition ---
|
# --- EFI/Boot Partition ---
|
||||||
echo " Installing GRUB..."
|
echo " Installing GRUB..."
|
||||||
|
|||||||
256
build/scripts/create-rpi-image.sh
Executable file
256
build/scripts/create-rpi-image.sh
Executable file
@@ -0,0 +1,256 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# create-rpi-image.sh — Create a raw disk image for Raspberry Pi SD card
|
||||||
|
#
|
||||||
|
# Partition layout (MBR):
|
||||||
|
# Part 1: Boot/Control (384 MB, FAT32, label KSOLOCTL) — firmware + kernel + initramfs + autoboot.txt
|
||||||
|
# Part 2: Boot A (256 MB, FAT32, label KSOLOA) — kernel + DTBs + initramfs
|
||||||
|
# Part 3: Boot B (256 MB, FAT32, label KSOLOB) — same as Boot A (initially identical)
|
||||||
|
# Part 4: Data (remaining of 2GB, ext4, label KSOLODATA)
|
||||||
|
#
|
||||||
|
# The RPi EEPROM loads start4.elf from partition 1.
|
||||||
|
# If autoboot.txt is supported (newer EEPROM), firmware redirects to partition 2/3 for A/B boot.
|
||||||
|
# If autoboot.txt is NOT supported (older EEPROM), partition 1 has full boot files as fallback.
|
||||||
|
#
|
||||||
|
# MBR is required — GPT + autoboot.txt is not reliably supported on Pi 4.
|
||||||
|
#
|
||||||
|
# Usage: build/scripts/create-rpi-image.sh
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
|
||||||
|
# shellcheck source=../config/versions.env
|
||||||
|
. "$SCRIPT_DIR/../config/versions.env"
|
||||||
|
|
||||||
|
ROOTFS_DIR="${ROOTFS_DIR:-$PROJECT_ROOT/build/rootfs-work}"
|
||||||
|
OUTPUT_DIR="${OUTPUT_DIR:-$PROJECT_ROOT/output}"
|
||||||
|
CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
||||||
|
VERSION="$(cat "$PROJECT_ROOT/VERSION")"
|
||||||
|
|
||||||
|
IMG_OUTPUT="$OUTPUT_DIR/${OS_NAME}-${VERSION}.rpi.img"
|
||||||
|
IMG_SIZE_MB="${IMG_SIZE_MB:-2048}" # 2 GB default
|
||||||
|
|
||||||
|
# ARM64 kernel (Image format, not bzImage)
|
||||||
|
KERNEL="${CACHE_DIR}/custom-kernel-arm64/Image"
|
||||||
|
INITRAMFS="${ROOTFS_DIR}/kubesolo-os.gz"
|
||||||
|
RPI_FIRMWARE_DIR="${CACHE_DIR}/rpi-firmware"
|
||||||
|
# DTBs MUST come from the kernel build (not firmware repo) to match the kernel.
|
||||||
|
# A DTB mismatch causes sdhci-iproc to silently fail — zero block devices.
|
||||||
|
KERNEL_DTBS_DIR="${CACHE_DIR}/custom-kernel-arm64/dtbs"
|
||||||
|
|
||||||
|
echo "==> Creating ${IMG_SIZE_MB}MB Raspberry Pi disk image..."
|
||||||
|
|
||||||
|
# --- Verify required files ---
|
||||||
|
MISSING=0
|
||||||
|
for f in "$KERNEL" "$INITRAMFS"; do
|
||||||
|
if [ ! -f "$f" ]; then
|
||||||
|
echo "ERROR: Missing $f"
|
||||||
|
MISSING=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ ! -d "$RPI_FIRMWARE_DIR" ]; then
|
||||||
|
echo "ERROR: Missing RPi firmware directory: $RPI_FIRMWARE_DIR"
|
||||||
|
echo " Run 'make fetch' to download firmware blobs."
|
||||||
|
MISSING=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$MISSING" = "1" ]; then
|
||||||
|
echo ""
|
||||||
|
echo "Required files:"
|
||||||
|
echo " Kernel: $KERNEL (run 'make kernel-arm64')"
|
||||||
|
echo " Initramfs: $INITRAMFS (run 'make initramfs')"
|
||||||
|
echo " Firmware: $RPI_FIRMWARE_DIR/ (run 'make fetch')"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "$OUTPUT_DIR"
|
||||||
|
|
||||||
|
# --- Create sparse image ---
|
||||||
|
dd if=/dev/zero of="$IMG_OUTPUT" bs=1M count=0 seek="$IMG_SIZE_MB" 2>/dev/null
|
||||||
|
|
||||||
|
# --- Partition table (MBR) ---
|
||||||
|
# MBR is required for reliable RPi boot with autoboot.txt.
|
||||||
|
# GPT + autoboot.txt fails on many Pi 4 EEPROM versions.
|
||||||
|
# Part 1: Boot/Control 384 MB FAT32 (firmware + kernel + initramfs + autoboot.txt)
|
||||||
|
# Part 2: Boot A 256 MB FAT32 (kernel + initramfs + DTBs)
|
||||||
|
# Part 3: Boot B 256 MB FAT32 (kernel + initramfs + DTBs)
|
||||||
|
# Part 4: Data remaining ext4
|
||||||
|
sfdisk "$IMG_OUTPUT" << EOF
|
||||||
|
label: dos
|
||||||
|
|
||||||
|
# Boot/Control partition: 384 MB, FAT32 (type 0c = W95 FAT32 LBA)
|
||||||
|
# Contains firmware + autoboot.txt for A/B redirect, PLUS full boot files as fallback
|
||||||
|
start=2048, size=786432, type=c, bootable
|
||||||
|
# Boot A partition: 256 MB, FAT32
|
||||||
|
size=524288, type=c
|
||||||
|
# Boot B partition: 256 MB, FAT32
|
||||||
|
size=524288, type=c
|
||||||
|
# Data partition: remaining, Linux
|
||||||
|
type=83
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# --- Set up loop device ---
|
||||||
|
LOOP=$(losetup --show -f "$IMG_OUTPUT")
|
||||||
|
echo "==> Loop device: $LOOP"
|
||||||
|
|
||||||
|
# Use kpartx for reliable partition device nodes (works in Docker/containers)
|
||||||
|
USE_KPARTX=false
|
||||||
|
if [ ! -b "${LOOP}p1" ]; then
|
||||||
|
if command -v kpartx >/dev/null 2>&1; then
|
||||||
|
kpartx -a "$LOOP"
|
||||||
|
USE_KPARTX=true
|
||||||
|
sleep 1
|
||||||
|
LOOP_NAME=$(basename "$LOOP")
|
||||||
|
P1="/dev/mapper/${LOOP_NAME}p1"
|
||||||
|
P2="/dev/mapper/${LOOP_NAME}p2"
|
||||||
|
P3="/dev/mapper/${LOOP_NAME}p3"
|
||||||
|
P4="/dev/mapper/${LOOP_NAME}p4"
|
||||||
|
else
|
||||||
|
# Retry with -P flag
|
||||||
|
losetup -d "$LOOP"
|
||||||
|
LOOP=$(losetup --show -fP "$IMG_OUTPUT")
|
||||||
|
sleep 1
|
||||||
|
P1="${LOOP}p1"
|
||||||
|
P2="${LOOP}p2"
|
||||||
|
P3="${LOOP}p3"
|
||||||
|
P4="${LOOP}p4"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
P1="${LOOP}p1"
|
||||||
|
P2="${LOOP}p2"
|
||||||
|
P3="${LOOP}p3"
|
||||||
|
P4="${LOOP}p4"
|
||||||
|
fi
|
||||||
|
|
||||||
|
MNT_CTL=$(mktemp -d)
|
||||||
|
MNT_BOOTA=$(mktemp -d)
|
||||||
|
MNT_BOOTB=$(mktemp -d)
|
||||||
|
MNT_DATA=$(mktemp -d)
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
umount "$MNT_CTL" 2>/dev/null || true
|
||||||
|
umount "$MNT_BOOTA" 2>/dev/null || true
|
||||||
|
umount "$MNT_BOOTB" 2>/dev/null || true
|
||||||
|
umount "$MNT_DATA" 2>/dev/null || true
|
||||||
|
if [ "$USE_KPARTX" = true ]; then
|
||||||
|
kpartx -d "$LOOP" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
losetup -d "$LOOP" 2>/dev/null || true
|
||||||
|
rm -rf "$MNT_CTL" "$MNT_BOOTA" "$MNT_BOOTB" "$MNT_DATA" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
# --- Format partitions ---
|
||||||
|
mkfs.vfat -F 32 -n KSOLOCTL "$P1"
|
||||||
|
mkfs.vfat -F 32 -n KSOLOA "$P2"
|
||||||
|
mkfs.vfat -F 32 -n KSOLOB "$P3"
|
||||||
|
mkfs.ext4 -q -L KSOLODATA "$P4"
|
||||||
|
|
||||||
|
# --- Mount all partitions ---
|
||||||
|
mount "$P1" "$MNT_CTL"
|
||||||
|
mount "$P2" "$MNT_BOOTA"
|
||||||
|
mount "$P3" "$MNT_BOOTB"
|
||||||
|
mount "$P4" "$MNT_DATA"
|
||||||
|
|
||||||
|
# --- Helper: populate a boot partition ---
|
||||||
|
populate_boot_partition() {
|
||||||
|
local MNT="$1"
|
||||||
|
local LABEL="$2"
|
||||||
|
|
||||||
|
echo " Populating $LABEL..."
|
||||||
|
|
||||||
|
# config.txt — Raspberry Pi boot configuration
|
||||||
|
cat > "$MNT/config.txt" << 'CFGTXT'
|
||||||
|
arm_64bit=1
|
||||||
|
kernel=kernel8.img
|
||||||
|
initramfs kubesolo-os.gz followkernel
|
||||||
|
enable_uart=1
|
||||||
|
gpu_mem=16
|
||||||
|
dtoverlay=disable-wifi
|
||||||
|
dtoverlay=disable-bt
|
||||||
|
CFGTXT
|
||||||
|
|
||||||
|
# cmdline.txt — kernel command line
|
||||||
|
# Note: must be a single line
|
||||||
|
echo "console=serial0,115200 console=tty1 kubesolo.data=LABEL=KSOLODATA quiet" > "$MNT/cmdline.txt"
|
||||||
|
|
||||||
|
# Copy kernel as kernel8.img (RPi 3/4/5 ARM64 convention)
|
||||||
|
cp "$KERNEL" "$MNT/kernel8.img"
|
||||||
|
|
||||||
|
# Copy initramfs
|
||||||
|
cp "$INITRAMFS" "$MNT/kubesolo-os.gz"
|
||||||
|
|
||||||
|
# Copy DTBs from kernel build (MUST match kernel to avoid driver probe failures)
|
||||||
|
if ls "$KERNEL_DTBS_DIR"/bcm27*.dtb 1>/dev/null 2>&1; then
|
||||||
|
cp "$KERNEL_DTBS_DIR"/bcm27*.dtb "$MNT/"
|
||||||
|
fi
|
||||||
|
# Copy overlays — prefer kernel-built, fall back to firmware repo
|
||||||
|
if [ -d "$KERNEL_DTBS_DIR/overlays" ]; then
|
||||||
|
cp -r "$KERNEL_DTBS_DIR/overlays" "$MNT/"
|
||||||
|
elif [ -d "$RPI_FIRMWARE_DIR/overlays" ]; then
|
||||||
|
cp -r "$RPI_FIRMWARE_DIR/overlays" "$MNT/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Write version marker
|
||||||
|
echo "$VERSION" > "$MNT/version.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Boot Control Partition (KSOLOCTL) ---
|
||||||
|
# Partition 1 serves dual purpose:
|
||||||
|
# 1. Contains firmware + autoboot.txt for A/B redirect (if EEPROM supports it)
|
||||||
|
# 2. Contains full boot files (kernel + initramfs) as fallback if autoboot.txt isn't supported
|
||||||
|
echo " Writing firmware + autoboot.txt + boot files to partition 1..."
|
||||||
|
|
||||||
|
# autoboot.txt — tells firmware which partition to boot from (A/B switching)
|
||||||
|
# If the EEPROM doesn't support this, it's silently ignored and the firmware
|
||||||
|
# falls back to booting from partition 1 using config.txt below.
|
||||||
|
cat > "$MNT_CTL/autoboot.txt" << 'AUTOBOOT'
|
||||||
|
[all]
|
||||||
|
tryboot_a_b=1
|
||||||
|
boot_partition=2
|
||||||
|
[tryboot]
|
||||||
|
boot_partition=3
|
||||||
|
AUTOBOOT
|
||||||
|
|
||||||
|
# Copy firmware blobs — REQUIRED on partition 1 for EEPROM to boot
|
||||||
|
if ls "$RPI_FIRMWARE_DIR"/start*.elf 1>/dev/null 2>&1; then
|
||||||
|
cp "$RPI_FIRMWARE_DIR"/start*.elf "$MNT_CTL/"
|
||||||
|
fi
|
||||||
|
if ls "$RPI_FIRMWARE_DIR"/fixup*.dat 1>/dev/null 2>&1; then
|
||||||
|
cp "$RPI_FIRMWARE_DIR"/fixup*.dat "$MNT_CTL/"
|
||||||
|
fi
|
||||||
|
if [ -f "$RPI_FIRMWARE_DIR/bootcode.bin" ]; then
|
||||||
|
cp "$RPI_FIRMWARE_DIR/bootcode.bin" "$MNT_CTL/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Full boot files on partition 1 — fallback if autoboot.txt redirect doesn't work.
|
||||||
|
# When autoboot.txt works, firmware switches to partition 2 and reads config.txt there.
|
||||||
|
# When autoboot.txt is unsupported, firmware reads THIS config.txt and boots from here.
|
||||||
|
populate_boot_partition "$MNT_CTL" "Boot Control (KSOLOCTL)"
|
||||||
|
|
||||||
|
# --- Boot A Partition (KSOLOA) ---
|
||||||
|
populate_boot_partition "$MNT_BOOTA" "Boot A (KSOLOA)"
|
||||||
|
|
||||||
|
# --- Boot B Partition (KSOLOB, initially identical) ---
|
||||||
|
populate_boot_partition "$MNT_BOOTB" "Boot B (KSOLOB)"
|
||||||
|
|
||||||
|
# --- Data Partition (KSOLODATA) ---
|
||||||
|
echo " Preparing data partition..."
|
||||||
|
for dir in kubesolo containerd etc-kubesolo log usr-local network images; do
|
||||||
|
mkdir -p "$MNT_DATA/$dir"
|
||||||
|
done
|
||||||
|
|
||||||
|
sync
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "==> Raspberry Pi disk image created: $IMG_OUTPUT"
|
||||||
|
echo " Size: $(du -h "$IMG_OUTPUT" | cut -f1)"
|
||||||
|
echo " Part 1 (KSOLOCTL): Firmware + kernel + initramfs + autoboot.txt (boot/control)"
|
||||||
|
echo " Part 2 (KSOLOA): Boot A — kernel + initramfs + DTBs"
|
||||||
|
echo " Part 3 (KSOLOB): Boot B — kernel + initramfs + DTBs"
|
||||||
|
echo " Part 4 (KSOLODATA): Persistent K8s state"
|
||||||
|
echo ""
|
||||||
|
echo "Write to SD card with:"
|
||||||
|
echo " sudo dd if=$IMG_OUTPUT of=/dev/sdX bs=4M status=progress"
|
||||||
|
echo ""
|
||||||
@@ -10,6 +10,111 @@ ROOTFS_DIR="${ROOTFS_DIR:-$PROJECT_ROOT/build/rootfs-work}"
|
|||||||
# shellcheck source=../config/versions.env
|
# shellcheck source=../config/versions.env
|
||||||
. "$SCRIPT_DIR/../config/versions.env"
|
. "$SCRIPT_DIR/../config/versions.env"
|
||||||
|
|
||||||
|
EXTRACT_ARCH="${TARGET_ARCH:-amd64}"
|
||||||
|
|
||||||
|
# Clean previous rootfs
|
||||||
|
rm -rf "$ROOTFS_DIR"
|
||||||
|
mkdir -p "$ROOTFS_DIR"
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# ARM64: piCore64 .img.gz extraction (SD card image, not ISO)
|
||||||
|
# =========================================================================
|
||||||
|
if [ "$EXTRACT_ARCH" = "arm64" ]; then
|
||||||
|
PICORE_IMG="$CACHE_DIR/$PICORE_IMAGE"
|
||||||
|
if [ ! -f "$PICORE_IMG" ]; then
|
||||||
|
echo "ERROR: piCore64 image not found: $PICORE_IMG"
|
||||||
|
echo "Run 'TARGET_ARCH=arm64 make fetch' first."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Extracting piCore64 image: $PICORE_IMG"
|
||||||
|
|
||||||
|
# Decompress to raw image (.img.gz or .zip)
|
||||||
|
PICORE_RAW="$CACHE_DIR/piCore-${PICORE_VERSION}.img"
|
||||||
|
if [ ! -f "$PICORE_RAW" ]; then
|
||||||
|
echo " Decompressing..."
|
||||||
|
case "$PICORE_IMG" in
|
||||||
|
*.zip)
|
||||||
|
unzip -o -j "$PICORE_IMG" '*.img' -d "$CACHE_DIR" 2>/dev/null || \
|
||||||
|
unzip -o "$PICORE_IMG" -d "$CACHE_DIR"
|
||||||
|
# Find the extracted .img file
|
||||||
|
EXTRACTED_IMG=$(find "$CACHE_DIR" -maxdepth 1 -name '*.img' -newer "$PICORE_IMG" | head -1)
|
||||||
|
if [ -n "$EXTRACTED_IMG" ] && [ "$EXTRACTED_IMG" != "$PICORE_RAW" ]; then
|
||||||
|
mv "$EXTRACTED_IMG" "$PICORE_RAW"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*.img.gz)
|
||||||
|
gunzip -k "$PICORE_IMG" 2>/dev/null || \
|
||||||
|
zcat "$PICORE_IMG" > "$PICORE_RAW"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: Unknown piCore image format: $PICORE_IMG"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Mount the piCore boot partition (partition 1) to find kernel/initramfs
|
||||||
|
# piCore layout: p1=boot (FAT32, has kernel+initramfs), p2=rootfs (ext4, has tce/)
|
||||||
|
IMG_MNT=$(mktemp -d)
|
||||||
|
echo " Mounting piCore boot partition..."
|
||||||
|
|
||||||
|
# Get partition 1 offset (boot/FAT partition with kernel+initramfs)
|
||||||
|
OFFSET=$(fdisk -l "$PICORE_RAW" 2>/dev/null | awk '/^.*img1/{print $2}')
|
||||||
|
if [ -z "$OFFSET" ]; then
|
||||||
|
# Fallback: try sfdisk (first partition)
|
||||||
|
OFFSET=$(sfdisk -d "$PICORE_RAW" 2>/dev/null | awk -F'[=,]' '/start=/{print $2; exit}' | tr -d ' ')
|
||||||
|
fi
|
||||||
|
if [ -z "$OFFSET" ]; then
|
||||||
|
echo "ERROR: Could not determine partition offset in piCore image"
|
||||||
|
fdisk -l "$PICORE_RAW" || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
BYTE_OFFSET=$((OFFSET * 512))
|
||||||
|
mount -o loop,ro,offset="$BYTE_OFFSET" "$PICORE_RAW" "$IMG_MNT" || {
|
||||||
|
echo "ERROR: Failed to mount piCore boot partition (need root for losetup)"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Find initramfs in the piCore boot partition
|
||||||
|
COREGZ=""
|
||||||
|
for f in "$IMG_MNT"/rootfs-piCore64*.gz "$IMG_MNT"/boot/corepure64.gz "$IMG_MNT"/boot/core.gz "$IMG_MNT"/corepure64.gz "$IMG_MNT"/core.gz; do
|
||||||
|
[ -f "$f" ] && COREGZ="$f" && break
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "$COREGZ" ]; then
|
||||||
|
echo "ERROR: Could not find initramfs in piCore image"
|
||||||
|
echo "Contents:"
|
||||||
|
ls -la "$IMG_MNT"/
|
||||||
|
ls -la "$IMG_MNT"/boot/ 2>/dev/null || true
|
||||||
|
umount "$IMG_MNT" 2>/dev/null || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Found initramfs: $COREGZ"
|
||||||
|
|
||||||
|
# Extract initramfs
|
||||||
|
mkdir -p "$ROOTFS_DIR/rootfs"
|
||||||
|
cd "$ROOTFS_DIR/rootfs"
|
||||||
|
zcat "$COREGZ" | cpio -idm 2>/dev/null
|
||||||
|
|
||||||
|
# Note: ARM64 kernel comes from build-kernel-arm64.sh, not from piCore
|
||||||
|
# We only use piCore for the BusyBox userland
|
||||||
|
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
umount "$IMG_MNT" 2>/dev/null || true
|
||||||
|
rm -rf "$IMG_MNT"
|
||||||
|
|
||||||
|
echo "==> ARM64 rootfs extracted: $ROOTFS_DIR/rootfs"
|
||||||
|
echo " Size: $(du -sh "$ROOTFS_DIR/rootfs" | cut -f1)"
|
||||||
|
echo "==> Extract complete (ARM64). Kernel will come from build-kernel-arm64.sh"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# x86_64: Tiny Core ISO extraction
|
||||||
|
# =========================================================================
|
||||||
TC_ISO="$CACHE_DIR/$TINYCORE_ISO"
|
TC_ISO="$CACHE_DIR/$TINYCORE_ISO"
|
||||||
ISO_MNT="$ROOTFS_DIR/iso-mount"
|
ISO_MNT="$ROOTFS_DIR/iso-mount"
|
||||||
|
|
||||||
@@ -19,9 +124,7 @@ if [ ! -f "$TC_ISO" ]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Clean previous rootfs
|
mkdir -p "$ISO_MNT"
|
||||||
rm -rf "$ROOTFS_DIR"
|
|
||||||
mkdir -p "$ROOTFS_DIR" "$ISO_MNT"
|
|
||||||
|
|
||||||
# --- Mount ISO and extract kernel + initramfs ---
|
# --- Mount ISO and extract kernel + initramfs ---
|
||||||
echo "==> Mounting ISO: $TC_ISO"
|
echo "==> Mounting ISO: $TC_ISO"
|
||||||
|
|||||||
@@ -10,9 +10,89 @@ CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
|||||||
# shellcheck source=../config/versions.env
|
# shellcheck source=../config/versions.env
|
||||||
. "$SCRIPT_DIR/../config/versions.env"
|
. "$SCRIPT_DIR/../config/versions.env"
|
||||||
|
|
||||||
|
# Verify SHA256 checksum of a downloaded file
|
||||||
|
verify_checksum() {
|
||||||
|
local file="$1" expected="$2" name="$3"
|
||||||
|
# Skip if no expected checksum provided
|
||||||
|
[ -z "$expected" ] && return 0
|
||||||
|
local actual
|
||||||
|
actual=$(sha256sum "$file" | awk '{print $1}')
|
||||||
|
if [ "$actual" = "$expected" ]; then
|
||||||
|
echo " Checksum OK: $name"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
echo "ERROR: Checksum mismatch for $name"
|
||||||
|
echo " Expected: $expected"
|
||||||
|
echo " Got: $actual"
|
||||||
|
rm -f "$file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
mkdir -p "$CACHE_DIR"
|
mkdir -p "$CACHE_DIR"
|
||||||
|
|
||||||
# --- Tiny Core Linux ISO ---
|
# Detect target architecture
|
||||||
|
FETCH_ARCH="${TARGET_ARCH:-amd64}"
|
||||||
|
|
||||||
|
# --- ARM64: piCore64 image instead of x86_64 ISO ---
|
||||||
|
if [ "$FETCH_ARCH" = "arm64" ]; then
|
||||||
|
PICORE_IMG="$CACHE_DIR/$PICORE_IMAGE"
|
||||||
|
if [ -f "$PICORE_IMG" ]; then
|
||||||
|
echo "==> piCore64 image already cached: $PICORE_IMG"
|
||||||
|
else
|
||||||
|
echo "==> Downloading piCore64 ${PICORE_VERSION} (${PICORE_ARCH})..."
|
||||||
|
echo " URL: $PICORE_IMAGE_URL"
|
||||||
|
wget -q --show-progress -O "$PICORE_IMG" "$PICORE_IMAGE_URL" 2>/dev/null || \
|
||||||
|
curl -fSL "$PICORE_IMAGE_URL" -o "$PICORE_IMG"
|
||||||
|
echo "==> Downloaded: $PICORE_IMG ($(du -h "$PICORE_IMG" | cut -f1))"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Also fetch RPi firmware
|
||||||
|
echo "==> Fetching RPi firmware..."
|
||||||
|
"$SCRIPT_DIR/fetch-rpi-firmware.sh"
|
||||||
|
|
||||||
|
# Download ARM64 KubeSolo binary
|
||||||
|
KUBESOLO_VERSION="${KUBESOLO_VERSION:-v1.1.0}"
|
||||||
|
KUBESOLO_BIN_ARM64="$CACHE_DIR/kubesolo-arm64"
|
||||||
|
if [ -f "$KUBESOLO_BIN_ARM64" ]; then
|
||||||
|
echo "==> KubeSolo ARM64 binary already cached: $KUBESOLO_BIN_ARM64"
|
||||||
|
else
|
||||||
|
echo "==> Downloading KubeSolo ${KUBESOLO_VERSION} (arm64)..."
|
||||||
|
BIN_URL="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-linux-arm64-musl.tar.gz"
|
||||||
|
BIN_URL_FALLBACK="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-linux-arm64.tar.gz"
|
||||||
|
TEMP_DIR=$(mktemp -d)
|
||||||
|
echo " URL: $BIN_URL"
|
||||||
|
if curl -fSL "$BIN_URL" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||||
|
echo " Downloaded musl variant (arm64)"
|
||||||
|
elif curl -fSL "$BIN_URL_FALLBACK" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||||
|
echo " Downloaded glibc variant (arm64 fallback)"
|
||||||
|
else
|
||||||
|
echo "ERROR: Failed to download KubeSolo ARM64 from GitHub."
|
||||||
|
rm -rf "$TEMP_DIR"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
tar -xzf "$TEMP_DIR/kubesolo.tar.gz" -C "$TEMP_DIR"
|
||||||
|
FOUND_BIN=$(find "$TEMP_DIR" -name "kubesolo" -type f ! -name "*.tar.gz" | head -1)
|
||||||
|
if [ -z "$FOUND_BIN" ]; then
|
||||||
|
echo "ERROR: Could not find kubesolo binary in extracted archive"
|
||||||
|
rm -rf "$TEMP_DIR"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
cp "$FOUND_BIN" "$KUBESOLO_BIN_ARM64"
|
||||||
|
chmod +x "$KUBESOLO_BIN_ARM64"
|
||||||
|
rm -rf "$TEMP_DIR"
|
||||||
|
echo "==> KubeSolo ARM64 binary: $KUBESOLO_BIN_ARM64 ($(du -h "$KUBESOLO_BIN_ARM64" | cut -f1))"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Skip x86_64 ISO and TCZ downloads for ARM64
|
||||||
|
echo ""
|
||||||
|
echo "==> ARM64 fetch complete."
|
||||||
|
echo "==> Component cache:"
|
||||||
|
ls -lh "$CACHE_DIR"/ 2>/dev/null || true
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- x86_64: Tiny Core Linux ISO ---
|
||||||
TC_ISO="$CACHE_DIR/$TINYCORE_ISO"
|
TC_ISO="$CACHE_DIR/$TINYCORE_ISO"
|
||||||
TC_URL="${TINYCORE_MIRROR}/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/release/${TINYCORE_ISO}"
|
TC_URL="${TINYCORE_MIRROR}/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/release/${TINYCORE_ISO}"
|
||||||
|
|
||||||
@@ -28,39 +108,134 @@ else
|
|||||||
wget -q --show-progress -O "$TC_ISO" "$TC_URL_ALT"
|
wget -q --show-progress -O "$TC_ISO" "$TC_URL_ALT"
|
||||||
}
|
}
|
||||||
echo "==> Downloaded: $TC_ISO ($(du -h "$TC_ISO" | cut -f1))"
|
echo "==> Downloaded: $TC_ISO ($(du -h "$TC_ISO" | cut -f1))"
|
||||||
|
verify_checksum "$TC_ISO" "$TINYCORE_ISO_SHA256" "Tiny Core ISO"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --- KubeSolo ---
|
# --- KubeSolo ---
|
||||||
KUBESOLO_INSTALLER="$CACHE_DIR/install-kubesolo.sh"
|
KUBESOLO_VERSION="${KUBESOLO_VERSION:-v1.1.0}"
|
||||||
KUBESOLO_BIN="$CACHE_DIR/kubesolo"
|
KUBESOLO_BIN="$CACHE_DIR/kubesolo"
|
||||||
|
|
||||||
if [ -f "$KUBESOLO_BIN" ]; then
|
if [ -f "$KUBESOLO_BIN" ]; then
|
||||||
echo "==> KubeSolo binary already cached: $KUBESOLO_BIN"
|
echo "==> KubeSolo binary already cached: $KUBESOLO_BIN"
|
||||||
else
|
else
|
||||||
echo "==> Downloading KubeSolo installer..."
|
echo "==> Downloading KubeSolo ${KUBESOLO_VERSION}..."
|
||||||
curl -sfL "$KUBESOLO_INSTALL_URL" -o "$KUBESOLO_INSTALLER"
|
|
||||||
|
|
||||||
echo "==> Extracting KubeSolo binary..."
|
# Determine architecture
|
||||||
echo " NOTE: The installer normally runs 'install'. We extract the binary URL instead."
|
ARCH="${TARGET_ARCH:-amd64}"
|
||||||
echo " For Phase 1 PoC, install KubeSolo on a host and copy the binary."
|
OS="linux"
|
||||||
|
|
||||||
|
# Build download URL from GitHub releases
|
||||||
|
# Available variants: kubesolo-v1.1.0-linux-amd64.tar.gz, kubesolo-v1.1.0-linux-amd64-musl.tar.gz
|
||||||
|
# We use the musl variant for maximum compatibility with Tiny Core Linux (musl-based)
|
||||||
|
BIN_URL="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-${OS}-${ARCH}-musl.tar.gz"
|
||||||
|
BIN_URL_FALLBACK="https://github.com/portainer/kubesolo/releases/download/${KUBESOLO_VERSION}/kubesolo-${KUBESOLO_VERSION}-${OS}-${ARCH}.tar.gz"
|
||||||
|
|
||||||
|
TEMP_DIR=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$TEMP_DIR"' EXIT
|
||||||
|
|
||||||
|
echo " URL: $BIN_URL"
|
||||||
|
if curl -fSL "$BIN_URL" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||||
|
echo " Downloaded musl variant"
|
||||||
|
elif curl -fSL "$BIN_URL_FALLBACK" -o "$TEMP_DIR/kubesolo.tar.gz" 2>/dev/null; then
|
||||||
|
echo " Downloaded glibc variant (fallback)"
|
||||||
|
else
|
||||||
|
echo "ERROR: Failed to download KubeSolo from GitHub."
|
||||||
|
echo " Tried: $BIN_URL"
|
||||||
|
echo " Tried: $BIN_URL_FALLBACK"
|
||||||
echo ""
|
echo ""
|
||||||
echo " Manual step required:"
|
echo " Manual step:"
|
||||||
echo " 1. On a Linux x86_64 host: curl -sfL https://get.kubesolo.io | sudo sh -"
|
echo " 1. Download from: https://github.com/portainer/kubesolo/releases"
|
||||||
echo " 2. Copy /usr/local/bin/kubesolo to: $KUBESOLO_BIN"
|
echo " 2. Extract and copy binary to: $KUBESOLO_BIN"
|
||||||
echo " 3. Re-run: make rootfs"
|
echo " 3. Re-run: make rootfs"
|
||||||
echo ""
|
exit 1
|
||||||
|
|
||||||
# Try to extract download URL from installer script
|
|
||||||
BINARY_URL=$(grep -oP 'https://[^ ]+kubesolo[^ ]+' "$KUBESOLO_INSTALLER" 2>/dev/null | head -1 || true)
|
|
||||||
if [ -n "$BINARY_URL" ]; then
|
|
||||||
echo " Attempting direct download from: $BINARY_URL"
|
|
||||||
curl -sfL "$BINARY_URL" -o "$KUBESOLO_BIN" && chmod +x "$KUBESOLO_BIN" || {
|
|
||||||
echo " Direct download failed. Use manual step above."
|
|
||||||
}
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -f "$KUBESOLO_BIN" ]; then
|
# Extract binary from tarball
|
||||||
|
tar -xzf "$TEMP_DIR/kubesolo.tar.gz" -C "$TEMP_DIR"
|
||||||
|
|
||||||
|
# Find the kubesolo binary in extracted contents
|
||||||
|
FOUND_BIN=$(find "$TEMP_DIR" -name "kubesolo" -type f ! -name "*.tar.gz" | head -1)
|
||||||
|
if [ -z "$FOUND_BIN" ]; then
|
||||||
|
echo "ERROR: Could not find kubesolo binary in extracted archive"
|
||||||
|
echo " Archive contents:"
|
||||||
|
ls -la "$TEMP_DIR"/
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cp "$FOUND_BIN" "$KUBESOLO_BIN"
|
||||||
|
chmod +x "$KUBESOLO_BIN"
|
||||||
|
|
||||||
|
trap - EXIT
|
||||||
|
rm -rf "$TEMP_DIR"
|
||||||
|
|
||||||
echo "==> KubeSolo binary: $KUBESOLO_BIN ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
echo "==> KubeSolo binary: $KUBESOLO_BIN ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
||||||
|
verify_checksum "$KUBESOLO_BIN" "$KUBESOLO_SHA256" "KubeSolo binary"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Tiny Core kernel module extensions (netfilter, iptables) ---
|
||||||
|
# The base Tiny Core initramfs does NOT include netfilter kernel modules.
|
||||||
|
# They are distributed as separate TCZ (squashfs) extensions.
|
||||||
|
# KubeSolo requires netfilter for kube-proxy, iptables NAT, conntrack, etc.
|
||||||
|
|
||||||
|
# Detect kernel version from the cached ISO
|
||||||
|
KVER=""
|
||||||
|
if [ -f "$TC_ISO" ]; then
|
||||||
|
# Try to detect kernel version from ISO without mounting
|
||||||
|
# Tiny Core 17.0 uses 6.18.2-tinycore64
|
||||||
|
KVER="6.18.2-tinycore64"
|
||||||
|
fi
|
||||||
|
|
||||||
|
NETFILTER_TCZ="$CACHE_DIR/ipv6-netfilter-${KVER}.tcz"
|
||||||
|
NETFILTER_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/ipv6-netfilter-${KVER}.tcz"
|
||||||
|
|
||||||
|
if [ -f "$NETFILTER_TCZ" ]; then
|
||||||
|
echo "==> Netfilter modules already cached: $NETFILTER_TCZ"
|
||||||
|
else
|
||||||
|
echo "==> Downloading netfilter kernel modules (ipv6-netfilter-${KVER}.tcz)..."
|
||||||
|
echo " URL: $NETFILTER_TCZ_URL"
|
||||||
|
if wget -q --show-progress -O "$NETFILTER_TCZ" "$NETFILTER_TCZ_URL" 2>/dev/null || \
|
||||||
|
curl -fSL "$NETFILTER_TCZ_URL" -o "$NETFILTER_TCZ" 2>/dev/null; then
|
||||||
|
echo "==> Downloaded: $NETFILTER_TCZ ($(du -h "$NETFILTER_TCZ" | cut -f1))"
|
||||||
|
verify_checksum "$NETFILTER_TCZ" "$NETFILTER_TCZ_SHA256" "netfilter TCZ"
|
||||||
|
else
|
||||||
|
echo "WARN: Failed to download netfilter modules. kube-proxy may not work."
|
||||||
|
rm -f "$NETFILTER_TCZ"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
NET_BRIDGING_TCZ="$CACHE_DIR/net-bridging-${KVER}.tcz"
|
||||||
|
NET_BRIDGING_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/net-bridging-${KVER}.tcz"
|
||||||
|
|
||||||
|
if [ -f "$NET_BRIDGING_TCZ" ]; then
|
||||||
|
echo "==> Net-bridging modules already cached: $NET_BRIDGING_TCZ"
|
||||||
|
else
|
||||||
|
echo "==> Downloading net-bridging kernel modules (net-bridging-${KVER}.tcz)..."
|
||||||
|
echo " URL: $NET_BRIDGING_TCZ_URL"
|
||||||
|
if wget -q --show-progress -O "$NET_BRIDGING_TCZ" "$NET_BRIDGING_TCZ_URL" 2>/dev/null || \
|
||||||
|
curl -fSL "$NET_BRIDGING_TCZ_URL" -o "$NET_BRIDGING_TCZ" 2>/dev/null; then
|
||||||
|
echo "==> Downloaded: $NET_BRIDGING_TCZ ($(du -h "$NET_BRIDGING_TCZ" | cut -f1))"
|
||||||
|
verify_checksum "$NET_BRIDGING_TCZ" "$NET_BRIDGING_TCZ_SHA256" "net-bridging TCZ"
|
||||||
|
else
|
||||||
|
echo "WARN: Failed to download net-bridging modules. CNI bridge may not work."
|
||||||
|
rm -f "$NET_BRIDGING_TCZ"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
IPTABLES_TCZ="$CACHE_DIR/iptables.tcz"
|
||||||
|
IPTABLES_TCZ_URL="https://distro.ibiblio.org/tinycorelinux/${TINYCORE_VERSION%%.*}.x/${TINYCORE_ARCH}/tcz/iptables.tcz"
|
||||||
|
|
||||||
|
if [ -f "$IPTABLES_TCZ" ]; then
|
||||||
|
echo "==> iptables userspace already cached: $IPTABLES_TCZ"
|
||||||
|
else
|
||||||
|
echo "==> Downloading iptables userspace tools..."
|
||||||
|
echo " URL: $IPTABLES_TCZ_URL"
|
||||||
|
if wget -q --show-progress -O "$IPTABLES_TCZ" "$IPTABLES_TCZ_URL" 2>/dev/null || \
|
||||||
|
curl -fSL "$IPTABLES_TCZ_URL" -o "$IPTABLES_TCZ" 2>/dev/null; then
|
||||||
|
echo "==> Downloaded: $IPTABLES_TCZ ($(du -h "$IPTABLES_TCZ" | cut -f1))"
|
||||||
|
verify_checksum "$IPTABLES_TCZ" "$IPTABLES_TCZ_SHA256" "iptables TCZ"
|
||||||
|
else
|
||||||
|
echo "WARN: Failed to download iptables. KubeSolo bundles its own but this is a fallback."
|
||||||
|
rm -f "$IPTABLES_TCZ"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
88
build/scripts/fetch-rpi-firmware.sh
Executable file
88
build/scripts/fetch-rpi-firmware.sh
Executable file
@@ -0,0 +1,88 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# fetch-rpi-firmware.sh — Download Raspberry Pi firmware blobs for boot
|
||||||
|
#
|
||||||
|
# Downloads firmware from the official raspberrypi/firmware GitHub repository.
|
||||||
|
# Extracts only the boot files needed: start*.elf, fixup*.dat, DTBs, bootcode.bin.
|
||||||
|
#
|
||||||
|
# Output: build/cache/rpi-firmware/ containing all required boot files.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
||||||
|
|
||||||
|
# shellcheck source=../config/versions.env
|
||||||
|
. "$SCRIPT_DIR/../config/versions.env"
|
||||||
|
|
||||||
|
RPI_FW_DIR="$CACHE_DIR/rpi-firmware"
|
||||||
|
RPI_FW_ARCHIVE="$CACHE_DIR/rpi-firmware-${RPI_FIRMWARE_TAG}.tar.gz"
|
||||||
|
|
||||||
|
# --- Skip if already fetched ---
|
||||||
|
if [ -d "$RPI_FW_DIR" ] && [ -f "$RPI_FW_DIR/start4.elf" ]; then
|
||||||
|
echo "==> RPi firmware already cached: $RPI_FW_DIR"
|
||||||
|
echo " Files: $(ls "$RPI_FW_DIR" | wc -l)"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Downloading Raspberry Pi firmware (tag: ${RPI_FIRMWARE_TAG})..."
|
||||||
|
mkdir -p "$CACHE_DIR" "$RPI_FW_DIR"
|
||||||
|
|
||||||
|
# --- Download firmware archive ---
|
||||||
|
if [ ! -f "$RPI_FW_ARCHIVE" ]; then
|
||||||
|
echo " URL: $RPI_FIRMWARE_URL"
|
||||||
|
wget -q --show-progress -O "$RPI_FW_ARCHIVE" "$RPI_FIRMWARE_URL" 2>/dev/null || \
|
||||||
|
curl -fSL "$RPI_FIRMWARE_URL" -o "$RPI_FW_ARCHIVE"
|
||||||
|
echo " Downloaded: $(du -h "$RPI_FW_ARCHIVE" | cut -f1)"
|
||||||
|
else
|
||||||
|
echo " Archive already cached: $(du -h "$RPI_FW_ARCHIVE" | cut -f1)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Extract boot files only ---
|
||||||
|
echo "==> Extracting boot files..."
|
||||||
|
TEMP_DIR=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$TEMP_DIR"' EXIT
|
||||||
|
|
||||||
|
# Extract only the boot/ directory from the archive
|
||||||
|
# Archive structure: firmware-<tag>/boot/...
|
||||||
|
tar -xzf "$RPI_FW_ARCHIVE" -C "$TEMP_DIR" --strip-components=1 --wildcards '*/boot/'
|
||||||
|
|
||||||
|
BOOT_SRC="$TEMP_DIR/boot"
|
||||||
|
if [ ! -d "$BOOT_SRC" ]; then
|
||||||
|
echo "ERROR: boot/ directory not found in firmware archive"
|
||||||
|
ls -la "$TEMP_DIR"/
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Copy GPU firmware (required for boot)
|
||||||
|
for f in "$BOOT_SRC"/start*.elf "$BOOT_SRC"/fixup*.dat; do
|
||||||
|
[ -f "$f" ] && cp "$f" "$RPI_FW_DIR/"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Copy bootcode.bin (first-stage boot for Pi 3 and older)
|
||||||
|
[ -f "$BOOT_SRC/bootcode.bin" ] && cp "$BOOT_SRC/bootcode.bin" "$RPI_FW_DIR/"
|
||||||
|
|
||||||
|
# Copy Device Tree Blobs for Pi 4 + Pi 5
|
||||||
|
for dtb in bcm2711-rpi-4-b.dtb bcm2711-rpi-400.dtb bcm2711-rpi-cm4.dtb \
|
||||||
|
bcm2712-rpi-5-b.dtb bcm2712d0-rpi-5-b.dtb; do
|
||||||
|
[ -f "$BOOT_SRC/$dtb" ] && cp "$BOOT_SRC/$dtb" "$RPI_FW_DIR/"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Copy overlays directory (needed for config.txt dtoverlay= directives)
|
||||||
|
if [ -d "$BOOT_SRC/overlays" ]; then
|
||||||
|
mkdir -p "$RPI_FW_DIR/overlays"
|
||||||
|
# Only copy overlays we actually use (disable-wifi, disable-bt)
|
||||||
|
for overlay in disable-wifi.dtbo disable-bt.dtbo; do
|
||||||
|
[ -f "$BOOT_SRC/overlays/$overlay" ] && \
|
||||||
|
cp "$BOOT_SRC/overlays/$overlay" "$RPI_FW_DIR/overlays/"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
trap - EXIT
|
||||||
|
rm -rf "$TEMP_DIR"
|
||||||
|
|
||||||
|
# --- Summary ---
|
||||||
|
echo ""
|
||||||
|
echo "==> RPi firmware extracted to: $RPI_FW_DIR"
|
||||||
|
echo " Files:"
|
||||||
|
ls -1 "$RPI_FW_DIR" | head -20
|
||||||
|
echo " Total size: $(du -sh "$RPI_FW_DIR" | cut -f1)"
|
||||||
@@ -8,6 +8,16 @@ CACHE_DIR="${CACHE_DIR:-$PROJECT_ROOT/build/cache}"
|
|||||||
ROOTFS_DIR="${ROOTFS_DIR:-$PROJECT_ROOT/build/rootfs-work}"
|
ROOTFS_DIR="${ROOTFS_DIR:-$PROJECT_ROOT/build/rootfs-work}"
|
||||||
ROOTFS="$ROOTFS_DIR/rootfs"
|
ROOTFS="$ROOTFS_DIR/rootfs"
|
||||||
VERSION="$(cat "$PROJECT_ROOT/VERSION")"
|
VERSION="$(cat "$PROJECT_ROOT/VERSION")"
|
||||||
|
INJECT_ARCH="${TARGET_ARCH:-amd64}"
|
||||||
|
|
||||||
|
# Architecture-specific paths
|
||||||
|
if [ "$INJECT_ARCH" = "arm64" ]; then
|
||||||
|
LIB_ARCH="aarch64-linux-gnu"
|
||||||
|
LD_SO="/lib/ld-linux-aarch64.so.1"
|
||||||
|
else
|
||||||
|
LIB_ARCH="x86_64-linux-gnu"
|
||||||
|
LD_SO="/lib64/ld-linux-x86-64.so.2"
|
||||||
|
fi
|
||||||
|
|
||||||
if [ ! -d "$ROOTFS" ]; then
|
if [ ! -d "$ROOTFS" ]; then
|
||||||
echo "ERROR: Rootfs not found: $ROOTFS"
|
echo "ERROR: Rootfs not found: $ROOTFS"
|
||||||
@@ -15,7 +25,11 @@ if [ ! -d "$ROOTFS" ]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ "$INJECT_ARCH" = "arm64" ]; then
|
||||||
|
KUBESOLO_BIN="$CACHE_DIR/kubesolo-arm64"
|
||||||
|
else
|
||||||
KUBESOLO_BIN="$CACHE_DIR/kubesolo"
|
KUBESOLO_BIN="$CACHE_DIR/kubesolo"
|
||||||
|
fi
|
||||||
if [ ! -f "$KUBESOLO_BIN" ]; then
|
if [ ! -f "$KUBESOLO_BIN" ]; then
|
||||||
echo "ERROR: KubeSolo binary not found: $KUBESOLO_BIN"
|
echo "ERROR: KubeSolo binary not found: $KUBESOLO_BIN"
|
||||||
echo "See fetch-components.sh output for instructions."
|
echo "See fetch-components.sh output for instructions."
|
||||||
@@ -25,15 +39,19 @@ fi
|
|||||||
echo "==> Injecting KubeSolo into rootfs..."
|
echo "==> Injecting KubeSolo into rootfs..."
|
||||||
|
|
||||||
# --- 1. KubeSolo binary ---
|
# --- 1. KubeSolo binary ---
|
||||||
mkdir -p "$ROOTFS/usr/local/bin"
|
# Install to /usr/bin (NOT /usr/local/bin) because /usr/local is bind-mounted
|
||||||
cp "$KUBESOLO_BIN" "$ROOTFS/usr/local/bin/kubesolo"
|
# from the data partition at boot, which would hide the binary.
|
||||||
chmod +x "$ROOTFS/usr/local/bin/kubesolo"
|
mkdir -p "$ROOTFS/usr/bin"
|
||||||
|
cp "$KUBESOLO_BIN" "$ROOTFS/usr/bin/kubesolo"
|
||||||
|
chmod +x "$ROOTFS/usr/bin/kubesolo"
|
||||||
echo " Installed KubeSolo binary ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
echo " Installed KubeSolo binary ($(du -h "$KUBESOLO_BIN" | cut -f1))"
|
||||||
|
|
||||||
# --- 2. Custom init system ---
|
# --- 2. Custom init system ---
|
||||||
echo " Installing init system..."
|
echo " Installing init system..."
|
||||||
|
|
||||||
# Main init
|
# Main init — remove symlink first to avoid clobbering busybox
|
||||||
|
# (Tiny Core has /sbin/init -> ../bin/busybox; cp follows symlinks)
|
||||||
|
rm -f "$ROOTFS/sbin/init"
|
||||||
cp "$PROJECT_ROOT/init/init.sh" "$ROOTFS/sbin/init"
|
cp "$PROJECT_ROOT/init/init.sh" "$ROOTFS/sbin/init"
|
||||||
chmod +x "$ROOTFS/sbin/init"
|
chmod +x "$ROOTFS/sbin/init"
|
||||||
|
|
||||||
@@ -64,31 +82,309 @@ for lib in network.sh health.sh; do
|
|||||||
done
|
done
|
||||||
|
|
||||||
# Cloud-init binary (Go, built separately)
|
# Cloud-init binary (Go, built separately)
|
||||||
CLOUDINIT_BIN="$CACHE_DIR/kubesolo-cloudinit"
|
# Try arch-specific binary first, then fall back to generic
|
||||||
|
CLOUDINIT_BIN="$CACHE_DIR/kubesolo-cloudinit-linux-$INJECT_ARCH"
|
||||||
|
[ ! -f "$CLOUDINIT_BIN" ] && CLOUDINIT_BIN="$CACHE_DIR/kubesolo-cloudinit"
|
||||||
if [ -f "$CLOUDINIT_BIN" ]; then
|
if [ -f "$CLOUDINIT_BIN" ]; then
|
||||||
cp "$CLOUDINIT_BIN" "$ROOTFS/usr/lib/kubesolo-os/kubesolo-cloudinit"
|
cp "$CLOUDINIT_BIN" "$ROOTFS/usr/lib/kubesolo-os/kubesolo-cloudinit"
|
||||||
chmod +x "$ROOTFS/usr/lib/kubesolo-os/kubesolo-cloudinit"
|
chmod +x "$ROOTFS/usr/lib/kubesolo-os/kubesolo-cloudinit"
|
||||||
echo " Installed cloud-init binary ($(du -h "$CLOUDINIT_BIN" | cut -f1))"
|
echo " Installed cloud-init binary ($(du -h "$CLOUDINIT_BIN" | cut -f1))"
|
||||||
else
|
else
|
||||||
echo " WARN: Cloud-init binary not found (run 'make build-cloudinit' to build)"
|
echo " WARN: Cloud-init binary not found (run 'make build-cloudinit' or 'make build-cross' to build)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Update agent binary (Go, built separately)
|
# Update agent binary (Go, built separately)
|
||||||
UPDATE_BIN="$CACHE_DIR/kubesolo-update"
|
# Try arch-specific binary first, then fall back to generic
|
||||||
|
UPDATE_BIN="$CACHE_DIR/kubesolo-update-linux-$INJECT_ARCH"
|
||||||
|
[ ! -f "$UPDATE_BIN" ] && UPDATE_BIN="$CACHE_DIR/kubesolo-update"
|
||||||
if [ -f "$UPDATE_BIN" ]; then
|
if [ -f "$UPDATE_BIN" ]; then
|
||||||
cp "$UPDATE_BIN" "$ROOTFS/usr/lib/kubesolo-os/kubesolo-update"
|
cp "$UPDATE_BIN" "$ROOTFS/usr/lib/kubesolo-os/kubesolo-update"
|
||||||
chmod +x "$ROOTFS/usr/lib/kubesolo-os/kubesolo-update"
|
chmod +x "$ROOTFS/usr/lib/kubesolo-os/kubesolo-update"
|
||||||
echo " Installed update agent ($(du -h "$UPDATE_BIN" | cut -f1))"
|
echo " Installed update agent ($(du -h "$UPDATE_BIN" | cut -f1))"
|
||||||
else
|
else
|
||||||
echo " WARN: Update agent not found (run 'make build-update-agent' to build)"
|
echo " WARN: Update agent not found (run 'make build-update-agent' or 'make build-cross' to build)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --- 3. Kernel modules list ---
|
# --- 3. Custom kernel or TCZ kernel modules ---
|
||||||
|
# If a custom kernel was built (with CONFIG_CGROUP_BPF=y), use it.
|
||||||
|
# Otherwise fall back to TCZ-extracted modules with manual modules.dep.
|
||||||
|
if [ "$INJECT_ARCH" = "arm64" ]; then
|
||||||
|
CUSTOM_KERNEL_DIR="$CACHE_DIR/custom-kernel-arm64"
|
||||||
|
CUSTOM_VMLINUZ="$CUSTOM_KERNEL_DIR/Image"
|
||||||
|
else
|
||||||
|
CUSTOM_KERNEL_DIR="$CACHE_DIR/custom-kernel"
|
||||||
|
CUSTOM_VMLINUZ="$CUSTOM_KERNEL_DIR/vmlinuz"
|
||||||
|
fi
|
||||||
|
CUSTOM_MODULES="$CUSTOM_KERNEL_DIR/modules"
|
||||||
|
|
||||||
|
# Detect kernel version from rootfs
|
||||||
|
KVER=""
|
||||||
|
for d in "$ROOTFS"/lib/modules/*/; do
|
||||||
|
[ -d "$d" ] && KVER="$(basename "$d")" && break
|
||||||
|
done
|
||||||
|
|
||||||
|
# Fallback: detect from custom kernel modules directory
|
||||||
|
if [ -z "$KVER" ] && [ -d "$CUSTOM_MODULES/lib/modules" ]; then
|
||||||
|
for d in "$CUSTOM_MODULES"/lib/modules/*/; do
|
||||||
|
[ -d "$d" ] && KVER="$(basename "$d")" && break
|
||||||
|
done
|
||||||
|
echo " Detected kernel version from custom kernel: $KVER"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$KVER" ]; then
|
||||||
|
echo " WARN: Could not detect kernel version from rootfs or custom kernel"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " Kernel version: $KVER"
|
||||||
|
|
||||||
|
if [ -f "$CUSTOM_VMLINUZ" ] && [ -d "$CUSTOM_MODULES/lib/modules/$KVER" ]; then
|
||||||
|
# =========================================================================
|
||||||
|
# Custom kernel path — selective module install (only what modules.list needs)
|
||||||
|
# =========================================================================
|
||||||
|
echo " Using custom kernel (CONFIG_CGROUP_BPF=y)..."
|
||||||
|
|
||||||
|
# Replace vmlinuz
|
||||||
|
cp "$CUSTOM_VMLINUZ" "$ROOTFS_DIR/vmlinuz"
|
||||||
|
echo " Installed custom vmlinuz ($(du -h "$CUSTOM_VMLINUZ" | cut -f1))"
|
||||||
|
|
||||||
|
# Selectively install ONLY modules from modules.list + their transitive deps.
|
||||||
|
# This keeps the initramfs minimal — no sound, GPU, SCSI, etc. modules.
|
||||||
|
echo " Installing kernel modules (selective — modules.list + deps only)..."
|
||||||
|
CUSTOM_MOD_DIR="$CUSTOM_MODULES/lib/modules/$KVER"
|
||||||
|
|
||||||
|
rm -rf "$ROOTFS/lib/modules/$KVER"
|
||||||
|
mkdir -p "$ROOTFS/lib/modules/$KVER/kernel"
|
||||||
|
|
||||||
|
# Copy module metadata files (needed by modprobe)
|
||||||
|
for f in modules.builtin modules.builtin.modinfo modules.order \
|
||||||
|
modules.builtin.alias.bin modules.builtin.bin; do
|
||||||
|
[ -f "$CUSTOM_MOD_DIR/$f" ] && cp "$CUSTOM_MOD_DIR/$f" "$ROOTFS/lib/modules/$KVER/"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Resolve and install modules from modules.list + transitive deps
|
||||||
|
if [ "$INJECT_ARCH" = "arm64" ]; then
|
||||||
|
MODULES_LIST="$PROJECT_ROOT/build/config/modules-arm64.list"
|
||||||
|
else
|
||||||
|
MODULES_LIST="$PROJECT_ROOT/build/config/modules.list"
|
||||||
|
fi
|
||||||
|
NEEDED_MODS=$(mktemp)
|
||||||
|
|
||||||
|
# Try modprobe first (works for same-arch builds)
|
||||||
|
MODPROBE_WORKS=true
|
||||||
|
FIRST_MOD=$(grep -v '^#' "$MODULES_LIST" | grep -v '^$' | head -1 | xargs)
|
||||||
|
if ! modprobe -S "$KVER" -d "$CUSTOM_MODULES" --show-depends "$FIRST_MOD" >/dev/null 2>&1; then
|
||||||
|
MODPROBE_WORKS=false
|
||||||
|
echo " modprobe cannot resolve modules (cross-arch build) — using find fallback"
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS= read -r mod; do
|
||||||
|
# Skip comments and blank lines
|
||||||
|
case "$mod" in \#*|"") continue ;; esac
|
||||||
|
mod=$(echo "$mod" | xargs) # trim whitespace
|
||||||
|
[ -z "$mod" ] && continue
|
||||||
|
|
||||||
|
if [ "$MODPROBE_WORKS" = true ]; then
|
||||||
|
# modprobe -S <ver> -d <root> --show-depends <module> lists all deps in load order
|
||||||
|
modprobe -S "$KVER" -d "$CUSTOM_MODULES" --show-depends "$mod" 2>/dev/null \
|
||||||
|
| awk '/^insmod/{print $2}' >> "$NEEDED_MODS" \
|
||||||
|
|| echo " WARN: modprobe could not resolve: $mod"
|
||||||
|
else
|
||||||
|
# Cross-arch fallback: find module by name in kernel tree
|
||||||
|
found=$(find "$CUSTOM_MOD_DIR/kernel" -name "${mod}.ko" -o -name "${mod}.ko.xz" -o -name "${mod}.ko.gz" -o -name "${mod}.ko.zst" 2>/dev/null | head -1)
|
||||||
|
if [ -n "$found" ]; then
|
||||||
|
echo "$found" >> "$NEEDED_MODS"
|
||||||
|
else
|
||||||
|
# Try replacing hyphens with underscores and vice versa
|
||||||
|
mod_alt=$(echo "$mod" | tr '-' '_')
|
||||||
|
found=$(find "$CUSTOM_MOD_DIR/kernel" -name "${mod_alt}.ko" -o -name "${mod_alt}.ko.xz" -o -name "${mod_alt}.ko.gz" -o -name "${mod_alt}.ko.zst" 2>/dev/null | head -1)
|
||||||
|
if [ -n "$found" ]; then
|
||||||
|
echo "$found" >> "$NEEDED_MODS"
|
||||||
|
else
|
||||||
|
echo " WARN: could not find module: $mod"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done < "$MODULES_LIST"
|
||||||
|
|
||||||
|
# Deduplicate and copy each needed module
|
||||||
|
sort -u "$NEEDED_MODS" | while IFS= read -r mod_path; do
|
||||||
|
mod_path=$(echo "$mod_path" | xargs) # trim whitespace
|
||||||
|
[ -z "$mod_path" ] && continue
|
||||||
|
# mod_path is absolute (e.g., /path/to/custom-kernel/modules/lib/modules/KVER/kernel/...)
|
||||||
|
if [ ! -f "$mod_path" ]; then
|
||||||
|
echo " WARN: module not found: $mod_path"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
# Get the relative path under lib/modules/KVER/
|
||||||
|
rel_path="${mod_path#$CUSTOM_MOD_DIR/}"
|
||||||
|
dst="$ROOTFS/lib/modules/$KVER/$rel_path"
|
||||||
|
mkdir -p "$(dirname "$dst")"
|
||||||
|
cp "$mod_path" "$dst"
|
||||||
|
done
|
||||||
|
rm -f "$NEEDED_MODS"
|
||||||
|
|
||||||
|
# Run depmod on the selective module set to generate correct metadata
|
||||||
|
depmod -a -b "$ROOTFS" "$KVER" 2>/dev/null || true
|
||||||
|
|
||||||
|
MOD_COUNT=$(find "$ROOTFS/lib/modules/$KVER" -name '*.ko*' | wc -l)
|
||||||
|
MOD_SIZE=$(du -sh "$ROOTFS/lib/modules/$KVER" | cut -f1)
|
||||||
|
echo " Installed $MOD_COUNT kernel modules ($MOD_SIZE) — minimal set"
|
||||||
|
|
||||||
|
else
|
||||||
|
# =========================================================================
|
||||||
|
# Stock kernel path — extract TCZ modules + manual modules.dep
|
||||||
|
# =========================================================================
|
||||||
|
echo " No custom kernel found, using stock kernel with TCZ modules..."
|
||||||
|
|
||||||
|
if [ -n "$KVER" ]; then
|
||||||
|
ROOTFS_MOD_DST="$ROOTFS/lib/modules/$KVER/kernel"
|
||||||
|
|
||||||
|
NETFILTER_TCZ="$CACHE_DIR/ipv6-netfilter-${KVER}.tcz"
|
||||||
|
if [ -f "$NETFILTER_TCZ" ]; then
|
||||||
|
echo " Extracting netfilter modules from $(basename "$NETFILTER_TCZ")..."
|
||||||
|
TCZ_TMP=$(mktemp -d)
|
||||||
|
if command -v unsquashfs >/dev/null 2>&1; then
|
||||||
|
unsquashfs -d "$TCZ_TMP/content" "$NETFILTER_TCZ" >/dev/null 2>&1
|
||||||
|
else
|
||||||
|
echo " ERROR: unsquashfs not found (install squashfs-tools)"
|
||||||
|
rm -rf "$TCZ_TMP"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
TCZ_MOD_SRC="$TCZ_TMP/content/usr/local/lib/modules/$KVER/kernel"
|
||||||
|
if [ -d "$TCZ_MOD_SRC" ]; then
|
||||||
|
find "$TCZ_MOD_SRC" -name '*.ko.gz' | while IFS= read -r mod_file; do
|
||||||
|
rel_path="${mod_file#$TCZ_MOD_SRC/}"
|
||||||
|
dst_dir="$ROOTFS_MOD_DST/$(dirname "$rel_path")"
|
||||||
|
mkdir -p "$dst_dir"
|
||||||
|
cp "$mod_file" "$dst_dir/"
|
||||||
|
done
|
||||||
|
MOD_COUNT=$(find "$TCZ_MOD_SRC" -name '*.ko.gz' | wc -l)
|
||||||
|
echo " Installed $MOD_COUNT kernel modules from netfilter TCZ"
|
||||||
|
fi
|
||||||
|
rm -rf "$TCZ_TMP"
|
||||||
|
else
|
||||||
|
echo " WARN: Netfilter TCZ not found. kube-proxy may not work."
|
||||||
|
fi
|
||||||
|
|
||||||
|
NET_BRIDGING_TCZ="$CACHE_DIR/net-bridging-${KVER}.tcz"
|
||||||
|
if [ -f "$NET_BRIDGING_TCZ" ]; then
|
||||||
|
echo " Extracting bridge modules from $(basename "$NET_BRIDGING_TCZ")..."
|
||||||
|
TCZ_TMP=$(mktemp -d)
|
||||||
|
unsquashfs -d "$TCZ_TMP/content" "$NET_BRIDGING_TCZ" >/dev/null 2>&1
|
||||||
|
TCZ_MOD_SRC="$TCZ_TMP/content/usr/local/lib/modules/$KVER/kernel"
|
||||||
|
if [ -d "$TCZ_MOD_SRC" ]; then
|
||||||
|
find "$TCZ_MOD_SRC" -name '*.ko.gz' | while IFS= read -r mod_file; do
|
||||||
|
rel_path="${mod_file#$TCZ_MOD_SRC/}"
|
||||||
|
dst_dir="$ROOTFS_MOD_DST/$(dirname "$rel_path")"
|
||||||
|
mkdir -p "$dst_dir"
|
||||||
|
cp "$mod_file" "$dst_dir/"
|
||||||
|
done
|
||||||
|
BR_COUNT=$(find "$TCZ_MOD_SRC" -name '*.ko.gz' | wc -l)
|
||||||
|
echo " Installed $BR_COUNT kernel modules from net-bridging TCZ"
|
||||||
|
fi
|
||||||
|
rm -rf "$TCZ_TMP"
|
||||||
|
else
|
||||||
|
echo " WARN: Net-bridging TCZ not found. CNI bridge networking may not work."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Manual modules.dep for stock kernel (Ubuntu's depmod can't handle TC's kernel)
|
||||||
|
MODULES_DEP="$ROOTFS/lib/modules/$KVER/modules.dep"
|
||||||
|
if [ -f "$MODULES_DEP" ]; then
|
||||||
|
echo " Appending module entries to modules.dep..."
|
||||||
|
cat >> "$MODULES_DEP" << 'MODDEP'
|
||||||
|
kernel/net/ipv6/ipv6.ko.gz:
|
||||||
|
kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko.gz:
|
||||||
|
kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko.gz: kernel/net/ipv6/ipv6.ko.gz
|
||||||
|
kernel/net/netfilter/nf_conntrack.ko.gz: kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko.gz kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko.gz
|
||||||
|
kernel/net/netfilter/nf_nat.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/nf_conntrack_netlink.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/nf_tables.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/nft_compat.ko.gz: kernel/net/netfilter/nf_tables.ko.gz
|
||||||
|
kernel/net/netfilter/nft_chain_nat.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz
|
||||||
|
kernel/net/netfilter/nft_ct.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/nft_masq.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/nft_nat.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz
|
||||||
|
kernel/net/netfilter/nft_redir.ko.gz: kernel/net/netfilter/nf_tables.ko.gz kernel/net/netfilter/nf_nat.ko.gz
|
||||||
|
kernel/net/netfilter/xt_conntrack.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/xt_MASQUERADE.ko.gz: kernel/net/netfilter/nf_nat.ko.gz kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/xt_mark.ko.gz:
|
||||||
|
kernel/net/netfilter/xt_comment.ko.gz:
|
||||||
|
kernel/net/netfilter/xt_multiport.ko.gz:
|
||||||
|
kernel/net/netfilter/xt_nat.ko.gz: kernel/net/netfilter/nf_nat.ko.gz
|
||||||
|
kernel/net/netfilter/xt_addrtype.ko.gz:
|
||||||
|
kernel/net/netfilter/xt_connmark.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz
|
||||||
|
kernel/net/netfilter/xt_REDIRECT.ko.gz: kernel/net/netfilter/nf_nat.ko.gz
|
||||||
|
kernel/net/netfilter/xt_recent.ko.gz:
|
||||||
|
kernel/net/netfilter/xt_statistic.ko.gz:
|
||||||
|
kernel/net/netfilter/xt_set.ko.gz: kernel/net/netfilter/ipset/ip_set.ko.gz
|
||||||
|
kernel/net/netfilter/ipset/ip_set.ko.gz:
|
||||||
|
kernel/net/ipv4/netfilter/nf_reject_ipv4.ko.gz:
|
||||||
|
kernel/net/ipv6/netfilter/nf_reject_ipv6.ko.gz:
|
||||||
|
kernel/net/ipv4/netfilter/ipt_REJECT.ko.gz: kernel/net/ipv4/netfilter/nf_reject_ipv4.ko.gz
|
||||||
|
kernel/net/ipv6/netfilter/ip6t_REJECT.ko.gz: kernel/net/ipv6/netfilter/nf_reject_ipv6.ko.gz
|
||||||
|
kernel/net/netfilter/nft_reject.ko.gz: kernel/net/netfilter/nf_tables.ko.gz
|
||||||
|
kernel/net/bridge/bridge.ko.gz: kernel/net/802/stp.ko.gz kernel/net/llc/llc.ko.gz
|
||||||
|
kernel/net/bridge/br_netfilter.ko.gz: kernel/net/bridge/bridge.ko.gz kernel/net/802/stp.ko.gz kernel/net/llc/llc.ko.gz
|
||||||
|
kernel/net/bridge/netfilter/nf_conntrack_bridge.ko.gz: kernel/net/netfilter/nf_conntrack.ko.gz kernel/net/bridge/bridge.ko.gz
|
||||||
|
MODDEP
|
||||||
|
|
||||||
|
find "$ROOTFS_MOD_DST" -name '*.ko.gz' -path '*/net/*' | sort | while IFS= read -r mod_file; do
|
||||||
|
rel_path="kernel/${mod_file#$ROOTFS_MOD_DST/}"
|
||||||
|
if ! grep -q "^${rel_path}:" "$MODULES_DEP" 2>/dev/null; then
|
||||||
|
echo "${rel_path}:" >> "$MODULES_DEP"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo " Updated modules.dep with netfilter entries"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Install iptables-nft (nftables-based iptables) from the builder system
|
||||||
|
# Kernel 6.18 uses nf_tables, not legacy ip_tables, so we need xtables-nft-multi
|
||||||
|
echo " Installing iptables-nft from builder..."
|
||||||
|
if [ -f /usr/sbin/xtables-nft-multi ]; then
|
||||||
|
mkdir -p "$ROOTFS/usr/sbin"
|
||||||
|
cp /usr/sbin/xtables-nft-multi "$ROOTFS/usr/sbin/"
|
||||||
|
|
||||||
|
# Create standard symlinks
|
||||||
|
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
|
||||||
|
ln -sf xtables-nft-multi "$ROOTFS/usr/sbin/$cmd"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Copy required shared libraries (architecture-aware paths)
|
||||||
|
mkdir -p "$ROOTFS/usr/lib/$LIB_ARCH" "$ROOTFS/lib/$LIB_ARCH"
|
||||||
|
[ "$INJECT_ARCH" != "arm64" ] && mkdir -p "$ROOTFS/lib64"
|
||||||
|
for lib in \
|
||||||
|
"/lib/$LIB_ARCH/libxtables.so.12"* \
|
||||||
|
"/lib/$LIB_ARCH/libmnl.so.0"* \
|
||||||
|
"/lib/$LIB_ARCH/libnftnl.so.11"* \
|
||||||
|
"/lib/$LIB_ARCH/libc.so.6" \
|
||||||
|
"$LD_SO"; do
|
||||||
|
[ -e "$lib" ] && cp -aL "$lib" "$ROOTFS${lib}" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
|
||||||
|
# Copy xtables modules directory (match extensions)
|
||||||
|
if [ -d "/usr/lib/$LIB_ARCH/xtables" ]; then
|
||||||
|
mkdir -p "$ROOTFS/usr/lib/$LIB_ARCH/xtables"
|
||||||
|
cp -a "/usr/lib/$LIB_ARCH/xtables/"*.so "$ROOTFS/usr/lib/$LIB_ARCH/xtables/" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " Installed iptables-nft (xtables-nft-multi) + shared libs"
|
||||||
|
else
|
||||||
|
echo " WARN: xtables-nft-multi not found in builder (install iptables package)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Kernel modules list (for init to load at boot)
|
||||||
|
if [ "$INJECT_ARCH" = "arm64" ]; then
|
||||||
|
cp "$PROJECT_ROOT/build/config/modules-arm64.list" "$ROOTFS/usr/lib/kubesolo-os/modules.list"
|
||||||
|
else
|
||||||
cp "$PROJECT_ROOT/build/config/modules.list" "$ROOTFS/usr/lib/kubesolo-os/modules.list"
|
cp "$PROJECT_ROOT/build/config/modules.list" "$ROOTFS/usr/lib/kubesolo-os/modules.list"
|
||||||
|
fi
|
||||||
|
|
||||||
# --- 4. Sysctl config ---
|
# --- 4. Sysctl config ---
|
||||||
mkdir -p "$ROOTFS/etc/sysctl.d"
|
mkdir -p "$ROOTFS/etc/sysctl.d"
|
||||||
cp "$PROJECT_ROOT/build/rootfs/etc/sysctl.d/k8s.conf" "$ROOTFS/etc/sysctl.d/k8s.conf"
|
cp "$PROJECT_ROOT/build/rootfs/etc/sysctl.d/k8s.conf" "$ROOTFS/etc/sysctl.d/k8s.conf"
|
||||||
|
cp "$PROJECT_ROOT/build/rootfs/etc/sysctl.d/security.conf" "$ROOTFS/etc/sysctl.d/security.conf"
|
||||||
|
|
||||||
# --- 5. OS metadata ---
|
# --- 5. OS metadata ---
|
||||||
echo "$VERSION" > "$ROOTFS/etc/kubesolo-os-version"
|
echo "$VERSION" > "$ROOTFS/etc/kubesolo-os-version"
|
||||||
@@ -120,7 +416,47 @@ mkdir -p "$ROOTFS/usr/local"
|
|||||||
mkdir -p "$ROOTFS/mnt/data"
|
mkdir -p "$ROOTFS/mnt/data"
|
||||||
mkdir -p "$ROOTFS/run/containerd"
|
mkdir -p "$ROOTFS/run/containerd"
|
||||||
|
|
||||||
# --- 8. Ensure /etc/hosts and /etc/resolv.conf exist ---
|
# --- 8. CA certificates (required for containerd to pull from registries) ---
|
||||||
|
mkdir -p "$ROOTFS/etc/ssl/certs"
|
||||||
|
if [ -f /etc/ssl/certs/ca-certificates.crt ]; then
|
||||||
|
cp /etc/ssl/certs/ca-certificates.crt "$ROOTFS/etc/ssl/certs/ca-certificates.crt"
|
||||||
|
echo " Installed CA certificates bundle"
|
||||||
|
elif [ -f /etc/pki/tls/certs/ca-bundle.crt ]; then
|
||||||
|
cp /etc/pki/tls/certs/ca-bundle.crt "$ROOTFS/etc/ssl/certs/ca-certificates.crt"
|
||||||
|
echo " Installed CA certificates bundle (from ca-bundle.crt)"
|
||||||
|
else
|
||||||
|
echo " WARN: No CA certificates found in builder — TLS verification will fail"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 9. AppArmor parser + profiles ---
|
||||||
|
echo " Installing AppArmor..."
|
||||||
|
if [ -f /usr/sbin/apparmor_parser ]; then
|
||||||
|
mkdir -p "$ROOTFS/usr/sbin"
|
||||||
|
cp /usr/sbin/apparmor_parser "$ROOTFS/usr/sbin/apparmor_parser"
|
||||||
|
chmod +x "$ROOTFS/usr/sbin/apparmor_parser"
|
||||||
|
|
||||||
|
# Copy shared libraries required by apparmor_parser
|
||||||
|
for lib in "/lib/$LIB_ARCH/libapparmor.so.1"*; do
|
||||||
|
[ -e "$lib" ] && cp -aL "$lib" "$ROOTFS${lib}" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
|
||||||
|
echo " Installed apparmor_parser + shared libs"
|
||||||
|
else
|
||||||
|
echo " WARN: apparmor_parser not found in builder (install apparmor package)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Copy AppArmor profiles
|
||||||
|
APPARMOR_PROFILES="$PROJECT_ROOT/build/rootfs/etc/apparmor.d"
|
||||||
|
if [ -d "$APPARMOR_PROFILES" ]; then
|
||||||
|
mkdir -p "$ROOTFS/etc/apparmor.d"
|
||||||
|
cp "$APPARMOR_PROFILES"/* "$ROOTFS/etc/apparmor.d/" 2>/dev/null || true
|
||||||
|
PROFILE_COUNT=$(ls "$ROOTFS/etc/apparmor.d/" 2>/dev/null | wc -l)
|
||||||
|
echo " Installed $PROFILE_COUNT AppArmor profiles"
|
||||||
|
else
|
||||||
|
echo " WARN: No AppArmor profiles found at $APPARMOR_PROFILES"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 10. Ensure /etc/hosts and /etc/resolv.conf exist ---
|
||||||
if [ ! -f "$ROOTFS/etc/hosts" ]; then
|
if [ ! -f "$ROOTFS/etc/hosts" ]; then
|
||||||
cat > "$ROOTFS/etc/hosts" << EOF
|
cat > "$ROOTFS/etc/hosts" << EOF
|
||||||
127.0.0.1 localhost
|
127.0.0.1 localhost
|
||||||
@@ -139,6 +475,6 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "==> Injection complete. Rootfs contents:"
|
echo "==> Injection complete. Rootfs contents:"
|
||||||
echo " Total size: $(du -sh "$ROOTFS" | cut -f1)"
|
echo " Total size: $(du -sh "$ROOTFS" | cut -f1)"
|
||||||
echo " KubeSolo: $(du -h "$ROOTFS/usr/local/bin/kubesolo" | cut -f1)"
|
echo " KubeSolo: $(du -h "$ROOTFS/usr/bin/kubesolo" | cut -f1)"
|
||||||
echo " Init stages: $(ls "$ROOTFS/usr/lib/kubesolo-os/init.d/" | wc -l)"
|
echo " Init stages: $(ls "$ROOTFS/usr/lib/kubesolo-os/init.d/" | wc -l)"
|
||||||
echo ""
|
echo ""
|
||||||
|
|||||||
@@ -33,7 +33,13 @@ type NetworkConfig struct {
|
|||||||
type KubeSoloConfig struct {
|
type KubeSoloConfig struct {
|
||||||
ExtraFlags string `yaml:"extra-flags"`
|
ExtraFlags string `yaml:"extra-flags"`
|
||||||
LocalStorage *bool `yaml:"local-storage"`
|
LocalStorage *bool `yaml:"local-storage"`
|
||||||
|
LocalStorageSharedPath string `yaml:"local-storage-shared-path"`
|
||||||
ExtraSANs []string `yaml:"apiserver-extra-sans"`
|
ExtraSANs []string `yaml:"apiserver-extra-sans"`
|
||||||
|
Debug bool `yaml:"debug"`
|
||||||
|
PprofServer bool `yaml:"pprof-server"`
|
||||||
|
PortainerEdgeID string `yaml:"portainer-edge-id"`
|
||||||
|
PortainerEdgeKey string `yaml:"portainer-edge-key"`
|
||||||
|
PortainerEdgeAsync bool `yaml:"portainer-edge-async"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// NTPConfig defines NTP settings.
|
// NTPConfig defines NTP settings.
|
||||||
|
|||||||
40
cloud-init/examples/full-config.yaml
Normal file
40
cloud-init/examples/full-config.yaml
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
# KubeSolo OS Cloud-Init — Full Configuration Reference
|
||||||
|
# Shows ALL supported KubeSolo parameters.
|
||||||
|
# Place at: /mnt/data/etc-kubesolo/cloud-init.yaml (on data partition)
|
||||||
|
# Or pass via boot param: kubesolo.cloudinit=/path/to/this.yaml
|
||||||
|
|
||||||
|
hostname: kubesolo-edge-01
|
||||||
|
|
||||||
|
network:
|
||||||
|
mode: dhcp
|
||||||
|
# interface: eth0 # Optional: specify interface (auto-detected if omitted)
|
||||||
|
# dns: # Optional: override DHCP-provided DNS
|
||||||
|
# - 8.8.8.8
|
||||||
|
|
||||||
|
kubesolo:
|
||||||
|
# Enable local-path-provisioner for persistent volumes (default: true)
|
||||||
|
local-storage: true
|
||||||
|
|
||||||
|
# Shared path for local-path-provisioner storage
|
||||||
|
local-storage-shared-path: "/mnt/shared"
|
||||||
|
|
||||||
|
# Extra SANs for API server TLS certificate
|
||||||
|
apiserver-extra-sans:
|
||||||
|
- kubesolo-edge-01.local
|
||||||
|
- 192.168.1.100
|
||||||
|
|
||||||
|
# Enable verbose debug logging
|
||||||
|
debug: false
|
||||||
|
|
||||||
|
# Enable Go pprof profiling server
|
||||||
|
pprof-server: false
|
||||||
|
|
||||||
|
# Portainer Edge Agent connection (alternative to portainer.edge-agent section)
|
||||||
|
# These generate --portainer-edge-id, --portainer-edge-key, --portainer-edge-async
|
||||||
|
# CLI flags for KubeSolo's built-in Edge Agent support.
|
||||||
|
portainer-edge-id: "your-edge-id"
|
||||||
|
portainer-edge-key: "your-edge-key"
|
||||||
|
portainer-edge-async: true
|
||||||
|
|
||||||
|
# Arbitrary extra flags passed directly to the KubeSolo binary
|
||||||
|
# extra-flags: "--disable traefik --disable servicelb"
|
||||||
@@ -46,6 +46,30 @@ func buildExtraFlags(cfg *Config) string {
|
|||||||
parts = append(parts, "--apiserver-extra-sans", san)
|
parts = append(parts, "--apiserver-extra-sans", san)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if cfg.KubeSolo.LocalStorageSharedPath != "" {
|
||||||
|
parts = append(parts, "--local-storage-shared-path", cfg.KubeSolo.LocalStorageSharedPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.KubeSolo.Debug {
|
||||||
|
parts = append(parts, "--debug")
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.KubeSolo.PprofServer {
|
||||||
|
parts = append(parts, "--pprof-server")
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.KubeSolo.PortainerEdgeID != "" {
|
||||||
|
parts = append(parts, "--portainer-edge-id", cfg.KubeSolo.PortainerEdgeID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.KubeSolo.PortainerEdgeKey != "" {
|
||||||
|
parts = append(parts, "--portainer-edge-key", cfg.KubeSolo.PortainerEdgeKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.KubeSolo.PortainerEdgeAsync {
|
||||||
|
parts = append(parts, "--portainer-edge-async")
|
||||||
|
}
|
||||||
|
|
||||||
return strings.Join(parts, " ")
|
return strings.Join(parts, " ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,54 @@ func TestBuildExtraFlags(t *testing.T) {
|
|||||||
},
|
},
|
||||||
want: "--disable servicelb --apiserver-extra-sans edge.local",
|
want: "--disable servicelb --apiserver-extra-sans edge.local",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "debug flag",
|
||||||
|
cfg: Config{
|
||||||
|
KubeSolo: KubeSoloConfig{Debug: true},
|
||||||
|
},
|
||||||
|
want: "--debug",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pprof-server flag",
|
||||||
|
cfg: Config{
|
||||||
|
KubeSolo: KubeSoloConfig{PprofServer: true},
|
||||||
|
},
|
||||||
|
want: "--pprof-server",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "local-storage-shared-path",
|
||||||
|
cfg: Config{
|
||||||
|
KubeSolo: KubeSoloConfig{LocalStorageSharedPath: "/mnt/shared"},
|
||||||
|
},
|
||||||
|
want: "--local-storage-shared-path /mnt/shared",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "portainer edge flags",
|
||||||
|
cfg: Config{
|
||||||
|
KubeSolo: KubeSoloConfig{
|
||||||
|
PortainerEdgeID: "test-id-123",
|
||||||
|
PortainerEdgeKey: "test-key-456",
|
||||||
|
PortainerEdgeAsync: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: "--portainer-edge-id test-id-123 --portainer-edge-key test-key-456 --portainer-edge-async",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "all new flags",
|
||||||
|
cfg: Config{
|
||||||
|
KubeSolo: KubeSoloConfig{
|
||||||
|
ExtraFlags: "--disable traefik",
|
||||||
|
ExtraSANs: []string{"node.local"},
|
||||||
|
LocalStorageSharedPath: "/mnt/data/shared",
|
||||||
|
Debug: true,
|
||||||
|
PprofServer: true,
|
||||||
|
PortainerEdgeID: "eid",
|
||||||
|
PortainerEdgeKey: "ekey",
|
||||||
|
PortainerEdgeAsync: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: "--disable traefik --apiserver-extra-sans node.local --local-storage-shared-path /mnt/data/shared --debug --pprof-server --portainer-edge-id eid --portainer-edge-key ekey --portainer-edge-async",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -64,6 +112,11 @@ func TestApplyKubeSolo(t *testing.T) {
|
|||||||
ExtraFlags: "--disable traefik",
|
ExtraFlags: "--disable traefik",
|
||||||
LocalStorage: &tr,
|
LocalStorage: &tr,
|
||||||
ExtraSANs: []string{"test.local"},
|
ExtraSANs: []string{"test.local"},
|
||||||
|
LocalStorageSharedPath: "/mnt/shared",
|
||||||
|
Debug: true,
|
||||||
|
PortainerEdgeID: "eid",
|
||||||
|
PortainerEdgeKey: "ekey",
|
||||||
|
PortainerEdgeAsync: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,6 +136,21 @@ func TestApplyKubeSolo(t *testing.T) {
|
|||||||
if !strings.Contains(flags, "--apiserver-extra-sans test.local") {
|
if !strings.Contains(flags, "--apiserver-extra-sans test.local") {
|
||||||
t.Errorf("extra-flags missing SANs: %q", flags)
|
t.Errorf("extra-flags missing SANs: %q", flags)
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(flags, "--local-storage-shared-path /mnt/shared") {
|
||||||
|
t.Errorf("extra-flags missing local-storage-shared-path: %q", flags)
|
||||||
|
}
|
||||||
|
if !strings.Contains(flags, "--debug") {
|
||||||
|
t.Errorf("extra-flags missing --debug: %q", flags)
|
||||||
|
}
|
||||||
|
if !strings.Contains(flags, "--portainer-edge-id eid") {
|
||||||
|
t.Errorf("extra-flags missing --portainer-edge-id: %q", flags)
|
||||||
|
}
|
||||||
|
if !strings.Contains(flags, "--portainer-edge-key ekey") {
|
||||||
|
t.Errorf("extra-flags missing --portainer-edge-key: %q", flags)
|
||||||
|
}
|
||||||
|
if !strings.Contains(flags, "--portainer-edge-async") {
|
||||||
|
t.Errorf("extra-flags missing --portainer-edge-async: %q", flags)
|
||||||
|
}
|
||||||
|
|
||||||
// Check config.yaml
|
// Check config.yaml
|
||||||
configData, err := os.ReadFile(filepath.Join(dir, "config.yaml"))
|
configData, err := os.ReadFile(filepath.Join(dir, "config.yaml"))
|
||||||
|
|||||||
@@ -225,6 +225,7 @@ func TestParseExampleFiles(t *testing.T) {
|
|||||||
"examples/static-ip.yaml",
|
"examples/static-ip.yaml",
|
||||||
"examples/portainer-edge.yaml",
|
"examples/portainer-edge.yaml",
|
||||||
"examples/airgapped.yaml",
|
"examples/airgapped.yaml",
|
||||||
|
"examples/full-config.yaml",
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, path := range examples {
|
for _, path := range examples {
|
||||||
|
|||||||
@@ -77,6 +77,21 @@ func buildEdgeAgentManifest(edgeID, edgeKey, portainerURL, image string) string
|
|||||||
sb.WriteString(" name: portainer-sa-clusteradmin\n")
|
sb.WriteString(" name: portainer-sa-clusteradmin\n")
|
||||||
sb.WriteString(" namespace: portainer\n")
|
sb.WriteString(" namespace: portainer\n")
|
||||||
sb.WriteString("---\n")
|
sb.WriteString("---\n")
|
||||||
|
sb.WriteString("apiVersion: v1\n")
|
||||||
|
sb.WriteString("kind: Service\n")
|
||||||
|
sb.WriteString("metadata:\n")
|
||||||
|
sb.WriteString(" name: portainer-agent\n")
|
||||||
|
sb.WriteString(" namespace: portainer\n")
|
||||||
|
sb.WriteString("spec:\n")
|
||||||
|
sb.WriteString(" clusterIP: None\n")
|
||||||
|
sb.WriteString(" selector:\n")
|
||||||
|
sb.WriteString(" app: portainer-agent\n")
|
||||||
|
sb.WriteString(" ports:\n")
|
||||||
|
sb.WriteString(" - name: agent\n")
|
||||||
|
sb.WriteString(" port: 9001\n")
|
||||||
|
sb.WriteString(" targetPort: 9001\n")
|
||||||
|
sb.WriteString(" protocol: TCP\n")
|
||||||
|
sb.WriteString("---\n")
|
||||||
sb.WriteString("apiVersion: apps/v1\n")
|
sb.WriteString("apiVersion: apps/v1\n")
|
||||||
sb.WriteString("kind: Deployment\n")
|
sb.WriteString("kind: Deployment\n")
|
||||||
sb.WriteString("metadata:\n")
|
sb.WriteString("metadata:\n")
|
||||||
|
|||||||
@@ -45,9 +45,15 @@ network:
|
|||||||
kubesolo:
|
kubesolo:
|
||||||
extra-flags: "--disable traefik" # Extra CLI flags for KubeSolo binary
|
extra-flags: "--disable traefik" # Extra CLI flags for KubeSolo binary
|
||||||
local-storage: true # Enable local-path provisioner (default: true)
|
local-storage: true # Enable local-path provisioner (default: true)
|
||||||
|
local-storage-shared-path: "/mnt/shared" # Shared path for local-path-provisioner
|
||||||
apiserver-extra-sans: # Extra SANs for API server certificate
|
apiserver-extra-sans: # Extra SANs for API server certificate
|
||||||
- node.example.com
|
- node.example.com
|
||||||
- 10.0.0.50
|
- 10.0.0.50
|
||||||
|
debug: false # Enable verbose debug logging
|
||||||
|
pprof-server: false # Enable Go pprof profiling server
|
||||||
|
portainer-edge-id: "" # Portainer Edge Agent ID
|
||||||
|
portainer-edge-key: "" # Portainer Edge Agent key
|
||||||
|
portainer-edge-async: false # Enable async Portainer Edge communication
|
||||||
|
|
||||||
# NTP servers (optional)
|
# NTP servers (optional)
|
||||||
ntp:
|
ntp:
|
||||||
@@ -129,6 +135,24 @@ kubesolo-cloudinit validate /path/to/cloud-init.yaml
|
|||||||
kubesolo-cloudinit dump /path/to/cloud-init.yaml
|
kubesolo-cloudinit dump /path/to/cloud-init.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## KubeSolo Configuration Reference
|
||||||
|
|
||||||
|
All fields under the `kubesolo:` section and their corresponding CLI flags:
|
||||||
|
|
||||||
|
| YAML Field | CLI Flag | Type | Default | Description |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `extra-flags` | (raw flags) | string | `""` | Arbitrary extra flags passed to KubeSolo binary |
|
||||||
|
| `local-storage` | `--local-storage` | bool | `true` | Enable local-path-provisioner for PVCs |
|
||||||
|
| `local-storage-shared-path` | `--local-storage-shared-path` | string | `""` | Shared path for local-path-provisioner storage |
|
||||||
|
| `apiserver-extra-sans` | `--apiserver-extra-sans` | list | `[]` | Extra SANs for API server TLS certificate |
|
||||||
|
| `debug` | `--debug` | bool | `false` | Enable verbose debug logging |
|
||||||
|
| `pprof-server` | `--pprof-server` | bool | `false` | Enable Go pprof profiling server |
|
||||||
|
| `portainer-edge-id` | `--portainer-edge-id` | string | `""` | Portainer Edge Agent ID (from Portainer UI) |
|
||||||
|
| `portainer-edge-key` | `--portainer-edge-key` | string | `""` | Portainer Edge Agent key (from Portainer UI) |
|
||||||
|
| `portainer-edge-async` | `--portainer-edge-async` | bool | `false` | Enable async Portainer Edge communication |
|
||||||
|
|
||||||
|
**Note:** The `portainer-edge-*` fields generate CLI flags for KubeSolo's built-in Edge Agent support. This is an alternative to the `portainer.edge-agent` section, which creates a standalone Kubernetes manifest. Use one approach or the other, not both.
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
See `cloud-init/examples/` for complete configuration examples:
|
See `cloud-init/examples/` for complete configuration examples:
|
||||||
@@ -137,6 +161,7 @@ See `cloud-init/examples/` for complete configuration examples:
|
|||||||
- `static-ip.yaml` — Static IP configuration
|
- `static-ip.yaml` — Static IP configuration
|
||||||
- `portainer-edge.yaml` — Portainer Edge Agent integration
|
- `portainer-edge.yaml` — Portainer Edge Agent integration
|
||||||
- `airgapped.yaml` — Air-gapped deployment with pre-loaded images
|
- `airgapped.yaml` — Air-gapped deployment with pre-loaded images
|
||||||
|
- `full-config.yaml` — All supported KubeSolo parameters
|
||||||
|
|
||||||
## Building
|
## Building
|
||||||
|
|
||||||
|
|||||||
100
hack/dev-vm-arm64.sh
Executable file
100
hack/dev-vm-arm64.sh
Executable file
@@ -0,0 +1,100 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# dev-vm-arm64.sh — Launch ARM64 QEMU VM for development
|
||||||
|
#
|
||||||
|
# Uses qemu-system-aarch64 with -machine virt to emulate an ARM64 system.
|
||||||
|
# This is useful for testing ARM64/RPi builds on x86_64 hosts.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./hack/dev-vm-arm64.sh # Use default kernel + initramfs
|
||||||
|
# ./hack/dev-vm-arm64.sh <kernel> <initramfs> # Specify custom paths
|
||||||
|
# ./hack/dev-vm-arm64.sh --debug # Enable debug logging
|
||||||
|
# ./hack/dev-vm-arm64.sh --shell # Drop to emergency shell
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
|
VMLINUZ=""
|
||||||
|
INITRD=""
|
||||||
|
EXTRA_APPEND=""
|
||||||
|
|
||||||
|
# Parse arguments
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--shell) EXTRA_APPEND="$EXTRA_APPEND kubesolo.shell" ;;
|
||||||
|
--debug) EXTRA_APPEND="$EXTRA_APPEND kubesolo.debug" ;;
|
||||||
|
*)
|
||||||
|
if [ -z "$VMLINUZ" ]; then
|
||||||
|
VMLINUZ="$arg"
|
||||||
|
elif [ -z "$INITRD" ]; then
|
||||||
|
INITRD="$arg"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Defaults
|
||||||
|
VMLINUZ="${VMLINUZ:-$PROJECT_ROOT/build/cache/custom-kernel-arm64/Image}"
|
||||||
|
INITRD="${INITRD:-$PROJECT_ROOT/build/rootfs-work/kubesolo-os.gz}"
|
||||||
|
|
||||||
|
# Verify files exist
|
||||||
|
if [ ! -f "$VMLINUZ" ]; then
|
||||||
|
echo "ERROR: Kernel not found: $VMLINUZ"
|
||||||
|
echo " Run 'make kernel-arm64' to build the ARM64 kernel."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -f "$INITRD" ]; then
|
||||||
|
echo "ERROR: Initrd not found: $INITRD"
|
||||||
|
echo " Run 'make initramfs' to build the initramfs."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Find mkfs.ext4
|
||||||
|
MKFS_EXT4=""
|
||||||
|
if command -v mkfs.ext4 >/dev/null 2>&1; then
|
||||||
|
MKFS_EXT4="mkfs.ext4"
|
||||||
|
elif [ -x "/opt/homebrew/opt/e2fsprogs/sbin/mkfs.ext4" ]; then
|
||||||
|
MKFS_EXT4="/opt/homebrew/opt/e2fsprogs/sbin/mkfs.ext4"
|
||||||
|
elif [ -x "/usr/local/opt/e2fsprogs/sbin/mkfs.ext4" ]; then
|
||||||
|
MKFS_EXT4="/usr/local/opt/e2fsprogs/sbin/mkfs.ext4"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$MKFS_EXT4" ]; then
|
||||||
|
echo "ERROR: mkfs.ext4 not found. Install e2fsprogs:"
|
||||||
|
if [ "$(uname)" = "Darwin" ]; then
|
||||||
|
echo " brew install e2fsprogs"
|
||||||
|
else
|
||||||
|
echo " apt install e2fsprogs # Debian/Ubuntu"
|
||||||
|
echo " dnf install e2fsprogs # Fedora/RHEL"
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create data disk
|
||||||
|
DATA_DISK="$(mktemp /tmp/kubesolo-arm64-data-XXXXXX).img"
|
||||||
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=1024 2>/dev/null
|
||||||
|
"$MKFS_EXT4" -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
trap 'rm -f "$DATA_DISK"' EXIT
|
||||||
|
|
||||||
|
echo "==> Launching ARM64 QEMU VM..."
|
||||||
|
echo " Kernel: $VMLINUZ"
|
||||||
|
echo " Initrd: $INITRD"
|
||||||
|
echo " Data: $DATA_DISK"
|
||||||
|
echo ""
|
||||||
|
echo " K8s API: localhost:6443"
|
||||||
|
echo " SSH: localhost:2222"
|
||||||
|
echo " Press Ctrl+A X to exit QEMU"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
qemu-system-aarch64 \
|
||||||
|
-machine virt \
|
||||||
|
-cpu cortex-a72 \
|
||||||
|
-m 2048 \
|
||||||
|
-smp 2 \
|
||||||
|
-nographic \
|
||||||
|
-kernel "$VMLINUZ" \
|
||||||
|
-initrd "$INITRD" \
|
||||||
|
-append "console=ttyAMA0 kubesolo.data=/dev/vda kubesolo.debug $EXTRA_APPEND" \
|
||||||
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
|
-net "nic,model=virtio" \
|
||||||
|
-net "user,hostfwd=tcp::6443-:6443,hostfwd=tcp::2222-:22"
|
||||||
159
hack/dev-vm.sh
159
hack/dev-vm.sh
@@ -1,24 +1,29 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# dev-vm.sh — Launch a QEMU VM for development and testing
|
# dev-vm.sh — Launch a QEMU VM for development and testing
|
||||||
# Usage: ./hack/dev-vm.sh [path-to-iso-or-img] [--shell] [--debug]
|
# Usage: ./hack/dev-vm.sh [path-to-iso-or-img] [--shell] [--debug]
|
||||||
|
#
|
||||||
|
# Works on both Linux (with KVM) and macOS (TCG emulation).
|
||||||
|
# On macOS/Apple Silicon, x86_64 guests run under TCG (~5-15x slower than KVM).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
VERSION="$(cat "$PROJECT_ROOT/VERSION")"
|
VERSION="$(cat "$PROJECT_ROOT/VERSION")"
|
||||||
|
ROOTFS_DIR="${ROOTFS_DIR:-$PROJECT_ROOT/build/rootfs-work}"
|
||||||
DEFAULT_ISO="$PROJECT_ROOT/output/kubesolo-os-${VERSION}.iso"
|
DEFAULT_ISO="$PROJECT_ROOT/output/kubesolo-os-${VERSION}.iso"
|
||||||
DEFAULT_IMG="$PROJECT_ROOT/output/kubesolo-os-${VERSION}.img"
|
DEFAULT_IMG="$PROJECT_ROOT/output/kubesolo-os-${VERSION}.img"
|
||||||
|
|
||||||
IMAGE="${1:-}"
|
IMAGE=""
|
||||||
EXTRA_APPEND=""
|
EXTRA_APPEND=""
|
||||||
SERIAL_OPTS="-serial stdio"
|
|
||||||
|
|
||||||
# Parse flags
|
# Parse all arguments — flags and optional image path
|
||||||
shift || true
|
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
case "$arg" in
|
case "$arg" in
|
||||||
--shell) EXTRA_APPEND="$EXTRA_APPEND kubesolo.shell" ;;
|
--shell) EXTRA_APPEND="$EXTRA_APPEND kubesolo.shell" ;;
|
||||||
--debug) EXTRA_APPEND="$EXTRA_APPEND kubesolo.debug" ;;
|
--debug) EXTRA_APPEND="$EXTRA_APPEND kubesolo.debug" ;;
|
||||||
|
--edge-id=*) EXTRA_APPEND="$EXTRA_APPEND kubesolo.edge_id=${arg#--edge-id=}" ;;
|
||||||
|
--edge-key=*) EXTRA_APPEND="$EXTRA_APPEND kubesolo.edge_key=${arg#--edge-key=}" ;;
|
||||||
|
*) IMAGE="$arg" ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -39,42 +44,146 @@ echo "==> Launching QEMU with: $IMAGE"
|
|||||||
echo " Press Ctrl+A, X to exit"
|
echo " Press Ctrl+A, X to exit"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
# Create a temporary data disk for persistence testing
|
DATA_APPEND=""
|
||||||
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
DATA_DISK=""
|
||||||
dd if=/dev/zero of="$DATA_DISK" bs=1M count=1024 2>/dev/null
|
|
||||||
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
|
||||||
|
|
||||||
cleanup() { rm -f "$DATA_DISK"; }
|
# Find mkfs.ext4 (Homebrew on macOS installs to a non-PATH location)
|
||||||
|
MKFS_EXT4=""
|
||||||
|
if command -v mkfs.ext4 >/dev/null 2>&1; then
|
||||||
|
MKFS_EXT4="mkfs.ext4"
|
||||||
|
elif [ -x "/opt/homebrew/opt/e2fsprogs/sbin/mkfs.ext4" ]; then
|
||||||
|
MKFS_EXT4="/opt/homebrew/opt/e2fsprogs/sbin/mkfs.ext4"
|
||||||
|
elif [ -x "/usr/local/opt/e2fsprogs/sbin/mkfs.ext4" ]; then
|
||||||
|
MKFS_EXT4="/usr/local/opt/e2fsprogs/sbin/mkfs.ext4"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create and attach a formatted data disk for persistent K8s state.
|
||||||
|
if [ -n "$MKFS_EXT4" ]; then
|
||||||
|
DATA_DISK="$(mktemp /tmp/kubesolo-data-XXXXXX).img"
|
||||||
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=2048 2>/dev/null
|
||||||
|
"$MKFS_EXT4" -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
DATA_APPEND="kubesolo.data=/dev/vda"
|
||||||
|
echo " Data disk: 2 GB ext4 (persistent)"
|
||||||
|
else
|
||||||
|
echo "ERROR: mkfs.ext4 not found. Install e2fsprogs:"
|
||||||
|
if [ "$(uname)" = "Darwin" ]; then
|
||||||
|
echo " brew install e2fsprogs"
|
||||||
|
else
|
||||||
|
echo " apt install e2fsprogs # Debian/Ubuntu"
|
||||||
|
echo " dnf install e2fsprogs # Fedora/RHEL"
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
[ -n "$DATA_DISK" ] && rm -f "$DATA_DISK" "${DATA_DISK%.img}"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
COMMON_OPTS=(
|
# Build QEMU command
|
||||||
-m 2048
|
QEMU_ARGS=(-m 2048 -smp 2 -nographic -cpu max)
|
||||||
-smp 2
|
QEMU_ARGS+=(-net "nic,model=virtio")
|
||||||
-nographic
|
QEMU_ARGS+=(-net "user,hostfwd=tcp::6443-:6443,hostfwd=tcp::2222-:22,hostfwd=tcp::8080-:8080")
|
||||||
-net nic,model=virtio
|
|
||||||
-net user,hostfwd=tcp::6443-:6443,hostfwd=tcp::2222-:22
|
|
||||||
-drive "file=$DATA_DISK,format=raw,if=virtio"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Enable KVM if available
|
if [ -n "$DATA_DISK" ]; then
|
||||||
|
QEMU_ARGS+=(-drive "file=$DATA_DISK,format=raw,if=virtio")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Enable KVM on Linux, fall back to TCG everywhere else
|
||||||
if [ -w /dev/kvm ] 2>/dev/null; then
|
if [ -w /dev/kvm ] 2>/dev/null; then
|
||||||
COMMON_OPTS+=(-enable-kvm)
|
QEMU_ARGS+=(-accel kvm)
|
||||||
echo " KVM acceleration: enabled"
|
echo " KVM acceleration: enabled"
|
||||||
else
|
else
|
||||||
echo " KVM acceleration: not available (using TCG)"
|
QEMU_ARGS+=(-accel tcg)
|
||||||
|
echo " TCG emulation (no KVM — expect slower boot)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
case "$IMAGE" in
|
case "$IMAGE" in
|
||||||
*.iso)
|
*.iso)
|
||||||
|
# -append only works with -kernel, not -cdrom.
|
||||||
|
# Extract kernel + initramfs and use direct kernel boot.
|
||||||
|
VMLINUZ=""
|
||||||
|
INITRAMFS=""
|
||||||
|
|
||||||
|
# Prefer build artifacts if present (no extraction needed)
|
||||||
|
if [ -f "$ROOTFS_DIR/vmlinuz" ] && [ -f "$ROOTFS_DIR/kubesolo-os.gz" ]; then
|
||||||
|
VMLINUZ="$ROOTFS_DIR/vmlinuz"
|
||||||
|
INITRAMFS="$ROOTFS_DIR/kubesolo-os.gz"
|
||||||
|
echo " Using kernel/initramfs from build directory"
|
||||||
|
else
|
||||||
|
# Extract kernel + initramfs from ISO.
|
||||||
|
# Try multiple methods: bsdtar > isoinfo > loop mount
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-extract-XXXXXX)"
|
||||||
|
EXTRACTED=0
|
||||||
|
|
||||||
|
echo " Extracting kernel/initramfs from ISO..."
|
||||||
|
|
||||||
|
# Method 1: bsdtar (ships with macOS, libarchive-tools on Linux)
|
||||||
|
if [ $EXTRACTED -eq 0 ] && command -v bsdtar >/dev/null 2>&1; then
|
||||||
|
if bsdtar -xf "$IMAGE" -C "$EXTRACT_DIR" boot/vmlinuz boot/kubesolo-os.gz 2>/dev/null; then
|
||||||
|
echo " Extracted via bsdtar"
|
||||||
|
EXTRACTED=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Method 2: isoinfo (genisoimage/cdrtools on Linux)
|
||||||
|
if [ $EXTRACTED -eq 0 ] && command -v isoinfo >/dev/null 2>&1; then
|
||||||
|
mkdir -p "$EXTRACT_DIR/boot"
|
||||||
|
isoinfo -i "$IMAGE" -x "/BOOT/VMLINUZ;1" > "$EXTRACT_DIR/boot/vmlinuz" 2>/dev/null || true
|
||||||
|
isoinfo -i "$IMAGE" -x "/BOOT/KUBESOLO-OS.GZ;1" > "$EXTRACT_DIR/boot/kubesolo-os.gz" 2>/dev/null || true
|
||||||
|
# isoinfo writes empty files on failure; check size
|
||||||
|
if [ -s "$EXTRACT_DIR/boot/vmlinuz" ] && [ -s "$EXTRACT_DIR/boot/kubesolo-os.gz" ]; then
|
||||||
|
echo " Extracted via isoinfo"
|
||||||
|
EXTRACTED=1
|
||||||
|
else
|
||||||
|
rm -f "$EXTRACT_DIR/boot/vmlinuz" "$EXTRACT_DIR/boot/kubesolo-os.gz"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Method 3: loop mount (Linux only, requires root)
|
||||||
|
if [ $EXTRACTED -eq 0 ] && [ "$(uname)" = "Linux" ]; then
|
||||||
|
ISO_MOUNT="$EXTRACT_DIR/mnt"
|
||||||
|
mkdir -p "$ISO_MOUNT"
|
||||||
|
if mount -o loop,ro "$IMAGE" "$ISO_MOUNT" 2>/dev/null; then
|
||||||
|
mkdir -p "$EXTRACT_DIR/boot"
|
||||||
|
cp "$ISO_MOUNT/boot/vmlinuz" "$EXTRACT_DIR/boot/" 2>/dev/null || true
|
||||||
|
cp "$ISO_MOUNT/boot/kubesolo-os.gz" "$EXTRACT_DIR/boot/" 2>/dev/null || true
|
||||||
|
umount "$ISO_MOUNT" 2>/dev/null || true
|
||||||
|
if [ -f "$EXTRACT_DIR/boot/vmlinuz" ] && [ -f "$EXTRACT_DIR/boot/kubesolo-os.gz" ]; then
|
||||||
|
echo " Extracted via loop mount"
|
||||||
|
EXTRACTED=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $EXTRACTED -eq 0 ]; then
|
||||||
|
echo "ERROR: Failed to extract kernel/initramfs from ISO."
|
||||||
|
echo " Install one of: bsdtar (libarchive-tools), isoinfo (genisoimage), or run as root for loop mount."
|
||||||
|
echo " Or run 'make rootfs initramfs' to produce build artifacts."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
VMLINUZ="$EXTRACT_DIR/boot/vmlinuz"
|
||||||
|
INITRAMFS="$EXTRACT_DIR/boot/kubesolo-os.gz"
|
||||||
|
|
||||||
|
if [ ! -f "$VMLINUZ" ] || [ ! -f "$INITRAMFS" ]; then
|
||||||
|
echo "ERROR: ISO does not contain expected boot/vmlinuz and boot/kubesolo-os.gz"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
qemu-system-x86_64 \
|
qemu-system-x86_64 \
|
||||||
"${COMMON_OPTS[@]}" \
|
"${QEMU_ARGS[@]}" \
|
||||||
-cdrom "$IMAGE" \
|
-kernel "$VMLINUZ" \
|
||||||
-boot d \
|
-initrd "$INITRAMFS" \
|
||||||
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda $EXTRA_APPEND"
|
-append "console=ttyS0,115200n8 $DATA_APPEND $EXTRA_APPEND"
|
||||||
;;
|
;;
|
||||||
*.img)
|
*.img)
|
||||||
qemu-system-x86_64 \
|
qemu-system-x86_64 \
|
||||||
"${COMMON_OPTS[@]}" \
|
"${QEMU_ARGS[@]}" \
|
||||||
-drive "file=$IMAGE,format=raw,if=virtio"
|
-drive "file=$IMAGE,format=raw,if=virtio"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
|
|||||||
48
hack/fix-portainer-service.sh
Executable file
48
hack/fix-portainer-service.sh
Executable file
@@ -0,0 +1,48 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# fix-portainer-service.sh — Create the missing headless Service for Portainer agent
|
||||||
|
# Usage: ./hack/fix-portainer-service.sh
|
||||||
|
#
|
||||||
|
# The Portainer agent does a DNS lookup for "portainer-agent" to discover peers.
|
||||||
|
# Without a Service, this lookup fails and the agent crashes.
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
KUBECONFIG_URL="http://localhost:8080"
|
||||||
|
|
||||||
|
echo "==> Fetching kubeconfig from $KUBECONFIG_URL..."
|
||||||
|
KUBECONFIG_FILE=$(mktemp)
|
||||||
|
trap 'rm -f "$KUBECONFIG_FILE"' EXIT
|
||||||
|
|
||||||
|
curl -s "$KUBECONFIG_URL" > "$KUBECONFIG_FILE"
|
||||||
|
|
||||||
|
if [ ! -s "$KUBECONFIG_FILE" ]; then
|
||||||
|
echo "ERROR: Failed to fetch kubeconfig. Is the VM running?"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Creating headless Service for portainer-agent..."
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_FILE" apply -f - <<'EOF'
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: portainer-agent
|
||||||
|
namespace: portainer
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
selector:
|
||||||
|
app: portainer-agent
|
||||||
|
ports:
|
||||||
|
- name: agent
|
||||||
|
port: 9001
|
||||||
|
targetPort: 9001
|
||||||
|
protocol: TCP
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "==> Restarting portainer-agent deployment..."
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_FILE" rollout restart -n portainer deployment/portainer-agent
|
||||||
|
|
||||||
|
echo "==> Waiting for rollout..."
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_FILE" rollout status -n portainer deployment/portainer-agent --timeout=120s
|
||||||
|
|
||||||
|
echo "==> Done. Checking pod status:"
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_FILE" get pods -n portainer
|
||||||
36
init/init.sh
36
init/init.sh
@@ -16,6 +16,39 @@
|
|||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
# --- Switch root: escape initramfs so runc pivot_root works ---
|
||||||
|
# The kernel boots into an initramfs (rootfs), which is a special mount that
|
||||||
|
# doesn't support pivot_root. Container runtimes (runc) need pivot_root to
|
||||||
|
# set up container root filesystems. To fix this, we copy the rootfs to a
|
||||||
|
# tmpfs and switch_root to it. The sentinel file prevents infinite loops.
|
||||||
|
if [ ! -f /etc/.switched_root ]; then
|
||||||
|
mount -t proc proc /proc 2>/dev/null || true
|
||||||
|
mount -t sysfs sysfs /sys 2>/dev/null || true
|
||||||
|
mount -t devtmpfs devtmpfs /dev 2>/dev/null || true
|
||||||
|
mkdir -p /mnt/newroot
|
||||||
|
mount -t tmpfs -o size=400M,mode=755 tmpfs /mnt/newroot
|
||||||
|
echo "[init] Copying rootfs to tmpfs..." >&2
|
||||||
|
# Copy each top-level directory explicitly (BusyBox cp -ax on rootfs is broken)
|
||||||
|
for d in bin sbin usr lib lib64 etc var opt; do
|
||||||
|
[ -d "/$d" ] && cp -a "/$d" /mnt/newroot/ 2>/dev/null || true
|
||||||
|
done
|
||||||
|
# Recreate mount point and special directories
|
||||||
|
mkdir -p /mnt/newroot/proc /mnt/newroot/sys /mnt/newroot/dev
|
||||||
|
mkdir -p /mnt/newroot/run /mnt/newroot/tmp /mnt/newroot/mnt
|
||||||
|
touch /mnt/newroot/etc/.switched_root
|
||||||
|
mount --move /proc /mnt/newroot/proc
|
||||||
|
mount --move /sys /mnt/newroot/sys
|
||||||
|
mount --move /dev /mnt/newroot/dev
|
||||||
|
echo "[init] Switching root..." >&2
|
||||||
|
exec switch_root /mnt/newroot /sbin/init
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- PATH setup ---
|
||||||
|
# Ensure /usr/local paths are in PATH (iptables, KubeSolo, etc.)
|
||||||
|
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||||
|
# iptables shared libraries live in /usr/local/lib
|
||||||
|
export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||||
|
|
||||||
# --- Constants ---
|
# --- Constants ---
|
||||||
INIT_LIB="/usr/lib/kubesolo-os"
|
INIT_LIB="/usr/lib/kubesolo-os"
|
||||||
INIT_STAGES="/usr/lib/kubesolo-os/init.d"
|
INIT_STAGES="/usr/lib/kubesolo-os/init.d"
|
||||||
@@ -29,6 +62,9 @@ export KUBESOLO_SHELL=""
|
|||||||
export KUBESOLO_NOPERSIST=""
|
export KUBESOLO_NOPERSIST=""
|
||||||
export KUBESOLO_CLOUDINIT=""
|
export KUBESOLO_CLOUDINIT=""
|
||||||
export KUBESOLO_EXTRA_FLAGS=""
|
export KUBESOLO_EXTRA_FLAGS=""
|
||||||
|
export KUBESOLO_PORTAINER_EDGE_ID=""
|
||||||
|
export KUBESOLO_PORTAINER_EDGE_KEY=""
|
||||||
|
export KUBESOLO_NOAPPARMOR=""
|
||||||
|
|
||||||
# --- Logging ---
|
# --- Logging ---
|
||||||
log() {
|
log() {
|
||||||
|
|||||||
@@ -1,23 +1,62 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# 00-early-mount.sh — Mount essential virtual filesystems
|
# 00-early-mount.sh — Mount essential virtual filesystems
|
||||||
|
|
||||||
|
# After switch_root, /proc /sys /dev are already mounted — only mount if missing
|
||||||
|
if ! mountpoint -q /proc 2>/dev/null; then
|
||||||
mount -t proc proc /proc 2>/dev/null || true
|
mount -t proc proc /proc 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if ! mountpoint -q /sys 2>/dev/null; then
|
||||||
mount -t sysfs sysfs /sys 2>/dev/null || true
|
mount -t sysfs sysfs /sys 2>/dev/null || true
|
||||||
mount -t devtmpfs devtmpfs /dev 2>/dev/null || true
|
fi
|
||||||
mount -t tmpfs tmpfs /tmp
|
if ! mountpoint -q /dev 2>/dev/null; then
|
||||||
mount -t tmpfs tmpfs /run
|
mount -t devtmpfs devtmpfs /dev 2>/dev/null || mount -t tmpfs tmpfs /dev
|
||||||
|
fi
|
||||||
|
if ! mountpoint -q /tmp 2>/dev/null; then
|
||||||
|
mount -t tmpfs -o noexec,nosuid,nodev,size=256M tmpfs /tmp
|
||||||
|
fi
|
||||||
|
if ! mountpoint -q /run 2>/dev/null; then
|
||||||
|
mount -t tmpfs -o nosuid,nodev,size=64M tmpfs /run
|
||||||
|
fi
|
||||||
|
|
||||||
mkdir -p /dev/pts /dev/shm
|
mkdir -p /dev/pts /dev/shm
|
||||||
|
if ! mountpoint -q /dev/pts 2>/dev/null; then
|
||||||
mount -t devpts devpts /dev/pts
|
mount -t devpts devpts /dev/pts
|
||||||
mount -t tmpfs tmpfs /dev/shm
|
fi
|
||||||
|
if ! mountpoint -q /dev/shm 2>/dev/null; then
|
||||||
|
mount -t tmpfs -o noexec,nosuid,nodev,size=64M tmpfs /dev/shm
|
||||||
|
fi
|
||||||
|
|
||||||
# Mount cgroup2 unified hierarchy
|
# Ensure essential device nodes exist (devtmpfs may be incomplete after switch_root)
|
||||||
|
[ -e /dev/console ] || mknod -m 600 /dev/console c 5 1 2>/dev/null || true
|
||||||
|
[ -e /dev/null ] || mknod -m 666 /dev/null c 1 3 2>/dev/null || true
|
||||||
|
[ -e /dev/zero ] || mknod -m 666 /dev/zero c 1 5 2>/dev/null || true
|
||||||
|
[ -e /dev/kmsg ] || mknod -m 660 /dev/kmsg c 1 11 2>/dev/null || true
|
||||||
|
[ -e /dev/random ] || mknod -m 666 /dev/random c 1 8 2>/dev/null || true
|
||||||
|
[ -e /dev/urandom ] || mknod -m 666 /dev/urandom c 1 9 2>/dev/null || true
|
||||||
|
[ -e /dev/tty ] || mknod -m 666 /dev/tty c 5 0 2>/dev/null || true
|
||||||
|
|
||||||
|
# Set up BusyBox mdev as hotplug handler (creates /dev nodes for new devices)
|
||||||
|
echo /sbin/mdev > /proc/sys/kernel/hotplug 2>/dev/null || true
|
||||||
|
mdev -s 2>/dev/null || true
|
||||||
|
|
||||||
|
# Mount cgroup v2 unified hierarchy
|
||||||
mkdir -p /sys/fs/cgroup
|
mkdir -p /sys/fs/cgroup
|
||||||
mount -t cgroup2 cgroup2 /sys/fs/cgroup 2>/dev/null || {
|
if ! mountpoint -q /sys/fs/cgroup 2>/dev/null; then
|
||||||
log_warn "cgroup v2 mount failed; attempting v1 fallback"
|
mount -t cgroup2 cgroup2 /sys/fs/cgroup 2>/dev/null || true
|
||||||
mount -t tmpfs cgroup /sys/fs/cgroup
|
fi
|
||||||
for subsys in cpu cpuacct memory devices freezer pids; do
|
|
||||||
mkdir -p "/sys/fs/cgroup/$subsys"
|
# Enable ALL available controllers for child cgroups
|
||||||
mount -t cgroup -o "$subsys" "cgroup_${subsys}" "/sys/fs/cgroup/$subsys" 2>/dev/null || true
|
# Required: memory (memory.max), cpu (cpu.max), pids (pids.max)
|
||||||
|
# First, move init process to its own cgroup so controllers can be enabled
|
||||||
|
# (cgroup v2 "no internal process" rule for non-root cgroups)
|
||||||
|
if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
|
||||||
|
mkdir -p /sys/fs/cgroup/init
|
||||||
|
echo $$ > /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true
|
||||||
|
for ctrl in $(cat /sys/fs/cgroup/cgroup.controllers); do
|
||||||
|
echo "+${ctrl}" > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null || true
|
||||||
done
|
done
|
||||||
}
|
fi
|
||||||
|
|
||||||
|
# Mount BPF filesystem (required for cgroup v2 device control via BPF)
|
||||||
|
mkdir -p /sys/fs/bpf
|
||||||
|
mount -t bpf bpf /sys/fs/bpf 2>/dev/null || true
|
||||||
|
|||||||
@@ -9,9 +9,16 @@ for arg in $(cat /proc/cmdline); do
|
|||||||
kubesolo.nopersist) KUBESOLO_NOPERSIST=1 ;;
|
kubesolo.nopersist) KUBESOLO_NOPERSIST=1 ;;
|
||||||
kubesolo.cloudinit=*) KUBESOLO_CLOUDINIT="${arg#kubesolo.cloudinit=}" ;;
|
kubesolo.cloudinit=*) KUBESOLO_CLOUDINIT="${arg#kubesolo.cloudinit=}" ;;
|
||||||
kubesolo.flags=*) KUBESOLO_EXTRA_FLAGS="${arg#kubesolo.flags=}" ;;
|
kubesolo.flags=*) KUBESOLO_EXTRA_FLAGS="${arg#kubesolo.flags=}" ;;
|
||||||
|
kubesolo.edge_id=*) KUBESOLO_PORTAINER_EDGE_ID="${arg#kubesolo.edge_id=}" ;;
|
||||||
|
kubesolo.edge_key=*) KUBESOLO_PORTAINER_EDGE_KEY="${arg#kubesolo.edge_key=}" ;;
|
||||||
|
kubesolo.nomodlock) KUBESOLO_NOMODLOCK=1 ;;
|
||||||
|
kubesolo.noapparmor) KUBESOLO_NOAPPARMOR=1 ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
|
export KUBESOLO_NOMODLOCK
|
||||||
|
export KUBESOLO_NOAPPARMOR
|
||||||
|
|
||||||
if [ -z "$KUBESOLO_DATA_DEV" ] && [ "$KUBESOLO_NOPERSIST" != "1" ]; then
|
if [ -z "$KUBESOLO_DATA_DEV" ] && [ "$KUBESOLO_NOPERSIST" != "1" ]; then
|
||||||
log_warn "No kubesolo.data= specified and kubesolo.nopersist not set"
|
log_warn "No kubesolo.data= specified and kubesolo.nopersist not set"
|
||||||
log_warn "Attempting auto-detection of data partition (label: KSOLODATA)"
|
log_warn "Attempting auto-detection of data partition (label: KSOLODATA)"
|
||||||
|
|||||||
@@ -8,25 +8,77 @@ if [ "$KUBESOLO_NOPERSIST" = "1" ]; then
|
|||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Wait for device to appear (USB, slow disks, virtio)
|
# Load block device drivers before waiting (modules loaded later in stage 30,
|
||||||
log "Waiting for data device: $KUBESOLO_DATA_DEV"
|
# but we need virtio_blk available NOW for /dev/vda detection)
|
||||||
|
modprobe virtio_blk 2>/dev/null || true
|
||||||
|
modprobe mmc_block 2>/dev/null || true
|
||||||
|
# Trigger mdev to create device nodes after loading driver
|
||||||
|
mdev -s 2>/dev/null || true
|
||||||
|
|
||||||
|
# Resolve LABEL= syntax to actual block device path
|
||||||
|
# The RPi cmdline uses kubesolo.data=LABEL=KSOLODATA which needs resolution
|
||||||
WAIT_SECS=30
|
WAIT_SECS=30
|
||||||
|
log "Waiting for data device: $KUBESOLO_DATA_DEV"
|
||||||
|
|
||||||
|
case "$KUBESOLO_DATA_DEV" in
|
||||||
|
LABEL=*)
|
||||||
|
# Extract label name and resolve via blkid/findfs
|
||||||
|
DATA_LABEL="${KUBESOLO_DATA_DEV#LABEL=}"
|
||||||
|
RESOLVED=""
|
||||||
for i in $(seq 1 "$WAIT_SECS"); do
|
for i in $(seq 1 "$WAIT_SECS"); do
|
||||||
[ -b "$KUBESOLO_DATA_DEV" ] && break
|
mdev -s 2>/dev/null || true
|
||||||
|
RESOLVED=$(blkid -L "$DATA_LABEL" 2>/dev/null) || true
|
||||||
|
if [ -z "$RESOLVED" ]; then
|
||||||
|
RESOLVED=$(findfs "LABEL=$DATA_LABEL" 2>/dev/null) || true
|
||||||
|
fi
|
||||||
|
if [ -n "$RESOLVED" ] && [ -b "$RESOLVED" ]; then
|
||||||
|
log "Resolved LABEL=$DATA_LABEL -> $RESOLVED"
|
||||||
|
KUBESOLO_DATA_DEV="$RESOLVED"
|
||||||
|
break
|
||||||
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
# Direct block device path — wait for it to appear
|
||||||
|
# Fallback: create device node from sysfs if devtmpfs/mdev didn't
|
||||||
|
DEV_NAME="${KUBESOLO_DATA_DEV##*/}"
|
||||||
|
if [ ! -b "$KUBESOLO_DATA_DEV" ] && [ -f "/sys/class/block/$DEV_NAME/dev" ]; then
|
||||||
|
MAJMIN=$(cat "/sys/class/block/$DEV_NAME/dev")
|
||||||
|
mknod "$KUBESOLO_DATA_DEV" b "${MAJMIN%%:*}" "${MAJMIN##*:}" 2>/dev/null || true
|
||||||
|
log "Created $KUBESOLO_DATA_DEV via mknod ($MAJMIN)"
|
||||||
|
fi
|
||||||
|
for i in $(seq 1 "$WAIT_SECS"); do
|
||||||
|
[ -b "$KUBESOLO_DATA_DEV" ] && break
|
||||||
|
mdev -s 2>/dev/null || true
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
if [ ! -b "$KUBESOLO_DATA_DEV" ]; then
|
if [ ! -b "$KUBESOLO_DATA_DEV" ]; then
|
||||||
log_err "Data device $KUBESOLO_DATA_DEV not found after ${WAIT_SECS}s"
|
log_err "Data device $KUBESOLO_DATA_DEV not found after ${WAIT_SECS}s"
|
||||||
|
# Show available block devices for debugging
|
||||||
|
log_err "Available block devices:"
|
||||||
|
ls -la /dev/mmc* /dev/sd* /dev/vd* 2>/dev/null | while read -r line; do
|
||||||
|
log_err " $line"
|
||||||
|
done
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Mount data partition
|
# Mount data partition (format on first boot if unformatted)
|
||||||
mkdir -p "$DATA_MOUNT"
|
mkdir -p "$DATA_MOUNT"
|
||||||
mount -t ext4 -o noatime "$KUBESOLO_DATA_DEV" "$DATA_MOUNT" || {
|
if ! mount -t ext4 -o noatime,nosuid,nodev "$KUBESOLO_DATA_DEV" "$DATA_MOUNT" 2>/dev/null; then
|
||||||
log_err "Failed to mount $KUBESOLO_DATA_DEV"
|
log "Formatting $KUBESOLO_DATA_DEV as ext4 (first boot)"
|
||||||
|
mkfs.ext4 -q -L KSOLODATA "$KUBESOLO_DATA_DEV" || {
|
||||||
|
log_err "Failed to format $KUBESOLO_DATA_DEV"
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
mount -t ext4 -o noatime,nosuid,nodev "$KUBESOLO_DATA_DEV" "$DATA_MOUNT" || {
|
||||||
|
log_err "Failed to mount $KUBESOLO_DATA_DEV after format"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fi
|
||||||
log_ok "Mounted $KUBESOLO_DATA_DEV at $DATA_MOUNT"
|
log_ok "Mounted $KUBESOLO_DATA_DEV at $DATA_MOUNT"
|
||||||
|
|
||||||
# Create persistent directory structure (first boot)
|
# Create persistent directory structure (first boot)
|
||||||
|
|||||||
47
init/lib/35-apparmor.sh
Normal file
47
init/lib/35-apparmor.sh
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# 35-apparmor.sh — Load AppArmor LSM profiles
|
||||||
|
|
||||||
|
# Check for opt-out boot parameter
|
||||||
|
if [ "$KUBESOLO_NOAPPARMOR" = "1" ]; then
|
||||||
|
log "AppArmor disabled via kubesolo.noapparmor boot parameter"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Mount securityfs if not already mounted
|
||||||
|
if ! mountpoint -q /sys/kernel/security 2>/dev/null; then
|
||||||
|
mount -t securityfs securityfs /sys/kernel/security 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if AppArmor is available in the kernel
|
||||||
|
if [ ! -d /sys/kernel/security/apparmor ]; then
|
||||||
|
log_warn "AppArmor not available in kernel — skipping profile loading"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for apparmor_parser
|
||||||
|
if ! command -v apparmor_parser >/dev/null 2>&1; then
|
||||||
|
log_warn "apparmor_parser not found — skipping profile loading"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Load all profiles from /etc/apparmor.d/
|
||||||
|
PROFILE_DIR="/etc/apparmor.d"
|
||||||
|
if [ ! -d "$PROFILE_DIR" ]; then
|
||||||
|
log_warn "No AppArmor profiles directory ($PROFILE_DIR) — skipping"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
LOADED=0
|
||||||
|
FAILED=0
|
||||||
|
|
||||||
|
for profile in "$PROFILE_DIR"/*; do
|
||||||
|
[ -f "$profile" ] || continue
|
||||||
|
if apparmor_parser -r "$profile" 2>/dev/null; then
|
||||||
|
LOADED=$((LOADED + 1))
|
||||||
|
else
|
||||||
|
log_warn "Failed to load AppArmor profile: $(basename "$profile")"
|
||||||
|
FAILED=$((FAILED + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
log_ok "AppArmor: loaded $LOADED profiles ($FAILED failed)"
|
||||||
@@ -18,16 +18,16 @@ fi
|
|||||||
# Fallback: DHCP on first non-loopback interface
|
# Fallback: DHCP on first non-loopback interface
|
||||||
log "Configuring network via DHCP"
|
log "Configuring network via DHCP"
|
||||||
|
|
||||||
# Bring up loopback
|
# Bring up loopback (use ifconfig for BusyBox compatibility)
|
||||||
ip link set lo up
|
ifconfig lo 127.0.0.1 netmask 255.0.0.0 up 2>/dev/null || \
|
||||||
ip addr add 127.0.0.1/8 dev lo
|
{ ip link set lo up 2>/dev/null && ip addr add 127.0.0.1/8 dev lo 2>/dev/null; } || true
|
||||||
|
|
||||||
# Find first ethernet interface
|
# Find first ethernet interface
|
||||||
ETH_DEV=""
|
ETH_DEV=""
|
||||||
for iface in /sys/class/net/*; do
|
for iface in /sys/class/net/*; do
|
||||||
iface="$(basename "$iface")"
|
iface="$(basename "$iface")"
|
||||||
case "$iface" in
|
case "$iface" in
|
||||||
lo|docker*|veth*|br*|cni*) continue ;;
|
lo|docker*|veth*|br*|cni*|dummy*|tunl*|sit*) continue ;;
|
||||||
esac
|
esac
|
||||||
ETH_DEV="$iface"
|
ETH_DEV="$iface"
|
||||||
break
|
break
|
||||||
@@ -39,7 +39,7 @@ if [ -z "$ETH_DEV" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
log "Using interface: $ETH_DEV"
|
log "Using interface: $ETH_DEV"
|
||||||
ip link set "$ETH_DEV" up
|
ifconfig "$ETH_DEV" up 2>/dev/null || ip link set "$ETH_DEV" up 2>/dev/null || true
|
||||||
|
|
||||||
# Run DHCP client (BusyBox udhcpc)
|
# Run DHCP client (BusyBox udhcpc)
|
||||||
if command -v udhcpc >/dev/null 2>&1; then
|
if command -v udhcpc >/dev/null 2>&1; then
|
||||||
@@ -58,4 +58,16 @@ else
|
|||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
log_ok "Network configured on $ETH_DEV"
|
# Ensure /etc/resolv.conf has valid DNS (udhcpc should have written it,
|
||||||
|
# but verify and add fallbacks if missing)
|
||||||
|
if [ ! -s /etc/resolv.conf ]; then
|
||||||
|
log_warn "/etc/resolv.conf is empty — adding fallback DNS"
|
||||||
|
echo "nameserver 10.0.2.3" > /etc/resolv.conf
|
||||||
|
echo "nameserver 8.8.8.8" >> /etc/resolv.conf
|
||||||
|
elif ! grep -q nameserver /etc/resolv.conf 2>/dev/null; then
|
||||||
|
log_warn "No nameserver in /etc/resolv.conf — adding fallback DNS"
|
||||||
|
echo "nameserver 10.0.2.3" >> /etc/resolv.conf
|
||||||
|
echo "nameserver 8.8.8.8" >> /etc/resolv.conf
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_ok "Network configured on $ETH_DEV (DNS: $(grep nameserver /etc/resolv.conf 2>/dev/null | head -1))"
|
||||||
|
|||||||
@@ -31,4 +31,17 @@ hostname "$HOSTNAME"
|
|||||||
echo "$HOSTNAME" > /etc/hostname
|
echo "$HOSTNAME" > /etc/hostname
|
||||||
echo "127.0.0.1 $HOSTNAME" >> /etc/hosts
|
echo "127.0.0.1 $HOSTNAME" >> /etc/hosts
|
||||||
|
|
||||||
|
# Generate /etc/machine-id if missing (kubelet requires it)
|
||||||
|
if [ ! -f /etc/machine-id ]; then
|
||||||
|
if [ -f "$DATA_MOUNT/etc-kubesolo/machine-id" ]; then
|
||||||
|
cp "$DATA_MOUNT/etc-kubesolo/machine-id" /etc/machine-id
|
||||||
|
else
|
||||||
|
# Generate from hostname hash (deterministic across reboots)
|
||||||
|
printf '%s' "$HOSTNAME" | md5sum 2>/dev/null | cut -d' ' -f1 > /etc/machine-id || \
|
||||||
|
cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' > /etc/machine-id || true
|
||||||
|
# Persist for next boot
|
||||||
|
cp /etc/machine-id "$DATA_MOUNT/etc-kubesolo/machine-id" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
log_ok "Hostname set to: $HOSTNAME"
|
log_ok "Hostname set to: $HOSTNAME"
|
||||||
|
|||||||
20
init/lib/85-security-lockdown.sh
Executable file
20
init/lib/85-security-lockdown.sh
Executable file
@@ -0,0 +1,20 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# 85-security-lockdown.sh — Lock down kernel after all modules loaded
|
||||||
|
|
||||||
|
# Allow disabling via boot parameter for debugging
|
||||||
|
if [ "$KUBESOLO_NOMODLOCK" = "1" ]; then
|
||||||
|
log_warn "Module lock DISABLED (kubesolo.nomodlock)"
|
||||||
|
else
|
||||||
|
# Permanently prevent new kernel module loading (irreversible until reboot)
|
||||||
|
# All required modules must already be loaded by stage 30
|
||||||
|
if [ -f /proc/sys/kernel/modules_disabled ]; then
|
||||||
|
echo 1 > /proc/sys/kernel/modules_disabled 2>/dev/null && \
|
||||||
|
log_ok "Kernel module loading locked" || \
|
||||||
|
log_warn "Failed to lock kernel module loading"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Safety net: enforce kernel information protection
|
||||||
|
# (also set via sysctl.d but enforce here in case sysctl.d was bypassed)
|
||||||
|
echo 2 > /proc/sys/kernel/kptr_restrict 2>/dev/null || true
|
||||||
|
echo 1 > /proc/sys/kernel/dmesg_restrict 2>/dev/null || true
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# 90-kubesolo.sh — Start KubeSolo (final init stage)
|
# 90-kubesolo.sh — Start KubeSolo (final init stage)
|
||||||
#
|
#
|
||||||
# This stage exec's KubeSolo as PID 1 (replacing init).
|
# Starts KubeSolo, waits for it to become ready, then prints the kubeconfig
|
||||||
# KubeSolo manages containerd, kubelet, API server, and all K8s components.
|
# to the console so it can be copied for remote kubectl access.
|
||||||
|
|
||||||
KUBESOLO_BIN="/usr/local/bin/kubesolo"
|
KUBESOLO_BIN="/usr/bin/kubesolo"
|
||||||
|
|
||||||
if [ ! -x "$KUBESOLO_BIN" ]; then
|
if [ ! -x "$KUBESOLO_BIN" ]; then
|
||||||
log_err "KubeSolo binary not found at $KUBESOLO_BIN"
|
log_err "KubeSolo binary not found at $KUBESOLO_BIN"
|
||||||
@@ -12,13 +12,15 @@ if [ ! -x "$KUBESOLO_BIN" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# Build KubeSolo command line
|
# Build KubeSolo command line
|
||||||
KUBESOLO_ARGS="--path /var/lib/kubesolo --local-storage true"
|
KUBESOLO_ARGS="--path /var/lib/kubesolo --local-storage"
|
||||||
|
|
||||||
# Add extra SANs if hostname resolves
|
# Add SANs for remote access (127.0.0.1 for QEMU port forwarding, 10.0.2.15 for QEMU NAT)
|
||||||
|
EXTRA_SANS="127.0.0.1,10.0.2.15"
|
||||||
HOSTNAME="$(hostname)"
|
HOSTNAME="$(hostname)"
|
||||||
if [ -n "$HOSTNAME" ]; then
|
if [ -n "$HOSTNAME" ]; then
|
||||||
KUBESOLO_ARGS="$KUBESOLO_ARGS --apiserver-extra-sans $HOSTNAME"
|
EXTRA_SANS="$EXTRA_SANS,$HOSTNAME"
|
||||||
fi
|
fi
|
||||||
|
KUBESOLO_ARGS="$KUBESOLO_ARGS --apiserver-extra-sans $EXTRA_SANS"
|
||||||
|
|
||||||
# Add any extra flags from boot parameters
|
# Add any extra flags from boot parameters
|
||||||
if [ -n "$KUBESOLO_EXTRA_FLAGS" ]; then
|
if [ -n "$KUBESOLO_EXTRA_FLAGS" ]; then
|
||||||
@@ -30,9 +32,81 @@ if [ -f /etc/kubesolo/extra-flags ]; then
|
|||||||
KUBESOLO_ARGS="$KUBESOLO_ARGS $(cat /etc/kubesolo/extra-flags)"
|
KUBESOLO_ARGS="$KUBESOLO_ARGS $(cat /etc/kubesolo/extra-flags)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
log "Starting KubeSolo: $KUBESOLO_BIN $KUBESOLO_ARGS"
|
# Pre-initialize iptables filter table and base chains.
|
||||||
log "Kubeconfig will be at: /var/lib/kubesolo/pki/admin/admin.kubeconfig"
|
# KubeSolo's kube-proxy uses iptables-restore (nf_tables backend) which needs
|
||||||
|
# the filter table to exist. Without this, the first iptables-restore fails
|
||||||
|
# with "RULE_APPEND failed (No such file or directory)".
|
||||||
|
if command -v iptables >/dev/null 2>&1; then
|
||||||
|
iptables -t filter -L -n >/dev/null 2>&1 || true
|
||||||
|
iptables -t nat -L -n >/dev/null 2>&1 || true
|
||||||
|
iptables -t mangle -L -n >/dev/null 2>&1 || true
|
||||||
|
log "Pre-initialized iptables tables (filter, nat, mangle)"
|
||||||
|
fi
|
||||||
|
|
||||||
# exec replaces this init process — KubeSolo becomes PID 1
|
# Export Portainer Edge env vars if set (via boot params or cloud-init)
|
||||||
|
if [ -n "${KUBESOLO_PORTAINER_EDGE_ID:-}" ]; then
|
||||||
|
export KUBESOLO_PORTAINER_EDGE_ID
|
||||||
|
log "Portainer Edge ID configured"
|
||||||
|
fi
|
||||||
|
if [ -n "${KUBESOLO_PORTAINER_EDGE_KEY:-}" ]; then
|
||||||
|
export KUBESOLO_PORTAINER_EDGE_KEY
|
||||||
|
log "Portainer Edge Key configured"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Starting KubeSolo: $KUBESOLO_BIN $KUBESOLO_ARGS"
|
||||||
|
|
||||||
|
KUBECONFIG_PATH="/var/lib/kubesolo/pki/admin/admin.kubeconfig"
|
||||||
|
|
||||||
|
# Start KubeSolo in background so we can wait for readiness and print kubeconfig
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
exec $KUBESOLO_BIN $KUBESOLO_ARGS
|
$KUBESOLO_BIN $KUBESOLO_ARGS &
|
||||||
|
KUBESOLO_PID=$!
|
||||||
|
|
||||||
|
# Wait for kubeconfig to appear (KubeSolo generates it during startup)
|
||||||
|
log "Waiting for KubeSolo to generate kubeconfig..."
|
||||||
|
WAIT=0
|
||||||
|
while [ ! -f "$KUBECONFIG_PATH" ] && [ $WAIT -lt 120 ]; do
|
||||||
|
sleep 2
|
||||||
|
WAIT=$((WAIT + 2))
|
||||||
|
# Check KubeSolo is still running
|
||||||
|
if ! kill -0 $KUBESOLO_PID 2>/dev/null; then
|
||||||
|
log_err "KubeSolo exited unexpectedly"
|
||||||
|
wait $KUBESOLO_PID 2>/dev/null || true
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -f "$KUBECONFIG_PATH" ]; then
|
||||||
|
log_ok "KubeSolo is running (PID $KUBESOLO_PID)"
|
||||||
|
|
||||||
|
# Rewrite server URL for external access and serve via HTTP.
|
||||||
|
# Serial console truncates long base64 cert lines, so we serve
|
||||||
|
# the kubeconfig over HTTP for reliable retrieval.
|
||||||
|
EXTERNAL_KC="/tmp/kubeconfig-external.yaml"
|
||||||
|
sed 's|server: https://.*:6443|server: https://localhost:6443|' "$KUBECONFIG_PATH" > "$EXTERNAL_KC"
|
||||||
|
|
||||||
|
# Serve kubeconfig via HTTP on port 8080 for remote kubectl access.
|
||||||
|
# Binds to 0.0.0.0 so it's reachable via QEMU port forwarding.
|
||||||
|
# Security: the kubeconfig is only useful if you can also reach
|
||||||
|
# port 6443 (API server). On edge devices, network isolation
|
||||||
|
# provides the security boundary.
|
||||||
|
(while true; do
|
||||||
|
printf 'HTTP/1.1 200 OK\r\nContent-Type: text/yaml\r\nConnection: close\r\n\r\n' | cat - "$EXTERNAL_KC" | nc -l -p 8080 2>/dev/null
|
||||||
|
done) &
|
||||||
|
|
||||||
|
log_ok "Kubeconfig available via HTTP on port 8080"
|
||||||
|
echo ""
|
||||||
|
echo "============================================================"
|
||||||
|
echo " From your host machine, run:"
|
||||||
|
echo ""
|
||||||
|
echo " curl -s http://localhost:8080 > ~/.kube/kubesolo-config"
|
||||||
|
echo " kubectl --kubeconfig ~/.kube/kubesolo-config get nodes"
|
||||||
|
echo "============================================================"
|
||||||
|
echo ""
|
||||||
|
else
|
||||||
|
log_warn "Kubeconfig not found after ${WAIT}s — KubeSolo may still be starting"
|
||||||
|
log_warn "Check manually: cat $KUBECONFIG_PATH"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Keep init alive — wait on KubeSolo process
|
||||||
|
wait $KUBESOLO_PID
|
||||||
|
|||||||
@@ -29,11 +29,16 @@ wait_for_file() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# Get IP address of an interface (POSIX-safe, no grep -P)
|
# Get IP address of an interface (BusyBox-safe: prefer ifconfig, fall back to ip)
|
||||||
get_iface_ip() {
|
get_iface_ip() {
|
||||||
iface="$1"
|
iface="$1"
|
||||||
|
if command -v ifconfig >/dev/null 2>&1; then
|
||||||
|
ifconfig "$iface" 2>/dev/null | \
|
||||||
|
sed -n 's/.*inet addr:\([0-9.]*\).*/\1/p;s/.*inet \([0-9.]*\).*/\1/p' | head -1
|
||||||
|
elif command -v ip >/dev/null 2>&1; then
|
||||||
ip -4 addr show "$iface" 2>/dev/null | \
|
ip -4 addr show "$iface" 2>/dev/null | \
|
||||||
sed -n 's/.*inet \([0-9.]*\).*/\1/p' | head -1
|
sed -n 's/.*inet \([0-9.]*\).*/\1/p' | head -1
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Check if running in a VM (useful for adjusting timeouts)
|
# Check if running in a VM (useful for adjusting timeouts)
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ RUNS=3
|
|||||||
SSH_PORT=2222
|
SSH_PORT=2222
|
||||||
K8S_PORT=6443
|
K8S_PORT=6443
|
||||||
|
|
||||||
|
. "$SCRIPT_DIR/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
shift || true
|
shift || true
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
@@ -47,6 +49,15 @@ echo "Type: $IMAGE_TYPE" >&2
|
|||||||
echo "Runs: $RUNS" >&2
|
echo "Runs: $RUNS" >&2
|
||||||
echo "" >&2
|
echo "" >&2
|
||||||
|
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
TEMP_DISK=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
[ -n "$TEMP_DISK" ] && rm -f "$TEMP_DISK"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
# Build QEMU command
|
# Build QEMU command
|
||||||
QEMU_CMD=(
|
QEMU_CMD=(
|
||||||
qemu-system-x86_64
|
qemu-system-x86_64
|
||||||
@@ -55,24 +66,31 @@ QEMU_CMD=(
|
|||||||
-nographic
|
-nographic
|
||||||
-no-reboot
|
-no-reboot
|
||||||
-serial mon:stdio
|
-serial mon:stdio
|
||||||
-net nic,model=virtio
|
-net "nic,model=virtio"
|
||||||
-net "user,hostfwd=tcp::${SSH_PORT}-:22,hostfwd=tcp::${K8S_PORT}-:6443"
|
-net "user,hostfwd=tcp::${SSH_PORT}-:22,hostfwd=tcp::${K8S_PORT}-:6443"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Add KVM if available
|
# Add KVM if available
|
||||||
if [ -e /dev/kvm ] && [ -r /dev/kvm ]; then
|
KVM_FLAG=$(detect_kvm)
|
||||||
|
if [ -n "$KVM_FLAG" ]; then
|
||||||
QEMU_CMD+=(-enable-kvm -cpu host)
|
QEMU_CMD+=(-enable-kvm -cpu host)
|
||||||
|
echo "KVM: enabled" >&2
|
||||||
else
|
else
|
||||||
QEMU_CMD+=(-cpu max)
|
QEMU_CMD+=(-cpu max)
|
||||||
|
echo "KVM: not available (TCG)" >&2
|
||||||
fi
|
fi
|
||||||
|
echo "" >&2
|
||||||
|
|
||||||
if [ "$IMAGE_TYPE" = "iso" ]; then
|
if [ "$IMAGE_TYPE" = "iso" ]; then
|
||||||
QEMU_CMD+=(-cdrom "$IMAGE")
|
# Extract kernel/initramfs for direct boot (required for -append to work)
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-bench-extract-XXXXXX)"
|
||||||
|
extract_kernel_from_iso "$IMAGE" "$EXTRACT_DIR" >&2
|
||||||
|
QEMU_CMD+=(-kernel "$VMLINUZ" -initrd "$INITRAMFS")
|
||||||
|
QEMU_CMD+=(-append "console=ttyS0,115200n8 kubesolo.debug")
|
||||||
# Add a temp disk for persistence
|
# Add a temp disk for persistence
|
||||||
TEMP_DISK=$(mktemp /tmp/kubesolo-bench-XXXXXX.img)
|
TEMP_DISK=$(mktemp /tmp/kubesolo-bench-XXXXXX.img)
|
||||||
qemu-img create -f qcow2 "$TEMP_DISK" 8G >/dev/null 2>&1
|
qemu-img create -f qcow2 "$TEMP_DISK" 8G >/dev/null 2>&1
|
||||||
QEMU_CMD+=(-drive "file=$TEMP_DISK,format=qcow2,if=virtio")
|
QEMU_CMD+=(-drive "file=$TEMP_DISK,format=qcow2,if=virtio")
|
||||||
trap "rm -f $TEMP_DISK" EXIT
|
|
||||||
else
|
else
|
||||||
QEMU_CMD+=(-drive "file=$IMAGE,format=raw,if=virtio")
|
QEMU_CMD+=(-drive "file=$IMAGE,format=raw,if=virtio")
|
||||||
fi
|
fi
|
||||||
@@ -111,7 +129,7 @@ for run in $(seq 1 "$RUNS"); do
|
|||||||
echo "KERNEL_MS=$ELAPSED_MS" >> "$LOG.times"
|
echo "KERNEL_MS=$ELAPSED_MS" >> "$LOG.times"
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
*"kubesolo-init"*"all stages complete"*|*"init complete"*)
|
*"KubeSolo is running"*|*"kubesolo-init"*"OK"*)
|
||||||
if [ -z "$INIT_DONE" ]; then
|
if [ -z "$INIT_DONE" ]; then
|
||||||
INIT_DONE="$ELAPSED_MS"
|
INIT_DONE="$ELAPSED_MS"
|
||||||
echo " Init complete: ${ELAPSED_MS}ms" >&2
|
echo " Init complete: ${ELAPSED_MS}ms" >&2
|
||||||
|
|||||||
@@ -5,42 +5,67 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
ISO="${1:?Usage: $0 <path-to-iso>}"
|
ISO="${1:?Usage: $0 <path-to-iso>}"
|
||||||
TIMEOUT_BOOT=120
|
TIMEOUT_K8S=${TIMEOUT_K8S:-300}
|
||||||
TIMEOUT_K8S=300
|
TIMEOUT_POD=${TIMEOUT_POD:-120}
|
||||||
TIMEOUT_POD=120
|
|
||||||
API_PORT=6443
|
API_PORT=6443
|
||||||
|
KC_PORT=8080
|
||||||
SERIAL_LOG=$(mktemp /tmp/kubesolo-workload-XXXXXX.log)
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-workload-XXXXXX.log)
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
. "$SCRIPT_DIR/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
||||||
dd if=/dev/zero of="$DATA_DISK" bs=1M count=1024 2>/dev/null
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=2048 2>/dev/null
|
||||||
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
|
||||||
|
QEMU_PID=""
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
KUBECONFIG_FILE=""
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
kill "$QEMU_PID" 2>/dev/null || true
|
[ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
|
||||||
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
||||||
|
[ -n "$KUBECONFIG_FILE" ] && rm -f "$KUBECONFIG_FILE"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
KUBECTL="kubectl --server=https://localhost:${API_PORT} --insecure-skip-tls-verify"
|
|
||||||
|
|
||||||
echo "==> Workload deployment test: $ISO"
|
echo "==> Workload deployment test: $ISO"
|
||||||
|
|
||||||
|
# Extract kernel from ISO
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-extract-XXXXXX)"
|
||||||
|
extract_kernel_from_iso "$ISO" "$EXTRACT_DIR"
|
||||||
|
|
||||||
|
KVM_FLAG=$(detect_kvm)
|
||||||
|
|
||||||
# Launch QEMU
|
# Launch QEMU
|
||||||
|
# shellcheck disable=SC2086
|
||||||
qemu-system-x86_64 \
|
qemu-system-x86_64 \
|
||||||
-m 2048 -smp 2 \
|
-m 2048 -smp 2 \
|
||||||
-nographic \
|
-nographic \
|
||||||
-cdrom "$ISO" \
|
$KVM_FLAG \
|
||||||
-boot d \
|
-kernel "$VMLINUZ" \
|
||||||
|
-initrd "$INITRAMFS" \
|
||||||
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
-net nic,model=virtio \
|
-net "nic,model=virtio" \
|
||||||
-net "user,hostfwd=tcp::${API_PORT}-:6443" \
|
-net "user,hostfwd=tcp::${API_PORT}-:6443,hostfwd=tcp::${KC_PORT}-:8080" \
|
||||||
-serial "file:$SERIAL_LOG" \
|
-serial "file:$SERIAL_LOG" \
|
||||||
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda" \
|
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug" \
|
||||||
&
|
&
|
||||||
QEMU_PID=$!
|
QEMU_PID=$!
|
||||||
|
|
||||||
|
# Wait for boot + fetch kubeconfig
|
||||||
|
echo " Waiting for boot..."
|
||||||
|
wait_for_boot "$SERIAL_LOG" "$QEMU_PID" 180 || exit 1
|
||||||
|
|
||||||
|
KUBECONFIG_FILE=$(mktemp /tmp/kubesolo-kubeconfig-XXXXXX.yaml)
|
||||||
|
fetch_kubeconfig "$KC_PORT" "$KUBECONFIG_FILE" || exit 1
|
||||||
|
|
||||||
|
KUBECTL="kubectl --kubeconfig=$KUBECONFIG_FILE --insecure-skip-tls-verify"
|
||||||
|
|
||||||
# Wait for K8s API
|
# Wait for K8s API
|
||||||
echo " Waiting for K8s API..."
|
echo " Waiting for K8s node Ready..."
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
K8S_READY=0
|
K8S_READY=0
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
||||||
@@ -71,6 +96,7 @@ $KUBECTL run test-nginx --image=nginx:alpine --restart=Never 2>/dev/null || {
|
|||||||
echo " Waiting for pod to reach Running..."
|
echo " Waiting for pod to reach Running..."
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
POD_RUNNING=0
|
POD_RUNNING=0
|
||||||
|
STATUS=""
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_POD" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_POD" ]; do
|
||||||
STATUS=$($KUBECTL get pod test-nginx -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
STATUS=$($KUBECTL get pod test-nginx -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
||||||
if [ "$STATUS" = "Running" ]; then
|
if [ "$STATUS" = "Running" ]; then
|
||||||
|
|||||||
@@ -5,58 +5,73 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
ISO="${1:?Usage: $0 <path-to-iso>}"
|
ISO="${1:?Usage: $0 <path-to-iso>}"
|
||||||
TIMEOUT_BOOT=120
|
TIMEOUT_K8S=${TIMEOUT_K8S:-300}
|
||||||
TIMEOUT_K8S=300
|
|
||||||
API_PORT=6443
|
API_PORT=6443
|
||||||
|
KC_PORT=8080
|
||||||
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-k8s-test-XXXXXX.log)
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
. "$SCRIPT_DIR/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
||||||
dd if=/dev/zero of="$DATA_DISK" bs=1M count=1024 2>/dev/null
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=2048 2>/dev/null
|
||||||
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
|
||||||
|
QEMU_PID=""
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
KUBECONFIG_FILE=""
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
kill "$QEMU_PID" 2>/dev/null || true
|
[ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
|
||||||
rm -f "$DATA_DISK"
|
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
||||||
|
[ -n "$KUBECONFIG_FILE" ] && rm -f "$KUBECONFIG_FILE"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
echo "==> K8s readiness test: $ISO"
|
echo "==> K8s readiness test: $ISO"
|
||||||
|
|
||||||
# Launch QEMU with API port forwarded
|
# Extract kernel from ISO
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-extract-XXXXXX)"
|
||||||
|
extract_kernel_from_iso "$ISO" "$EXTRACT_DIR"
|
||||||
|
|
||||||
|
KVM_FLAG=$(detect_kvm)
|
||||||
|
[ -n "$KVM_FLAG" ] && echo " KVM acceleration: enabled"
|
||||||
|
|
||||||
|
# Launch QEMU with API + kubeconfig ports forwarded
|
||||||
|
# shellcheck disable=SC2086
|
||||||
qemu-system-x86_64 \
|
qemu-system-x86_64 \
|
||||||
-m 2048 -smp 2 \
|
-m 2048 -smp 2 \
|
||||||
-nographic \
|
-nographic \
|
||||||
-cdrom "$ISO" \
|
$KVM_FLAG \
|
||||||
-boot d \
|
-kernel "$VMLINUZ" \
|
||||||
|
-initrd "$INITRAMFS" \
|
||||||
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
-net nic,model=virtio \
|
-net "nic,model=virtio" \
|
||||||
-net user,hostfwd=tcp::${API_PORT}-:6443 \
|
-net "user,hostfwd=tcp::${API_PORT}-:6443,hostfwd=tcp::${KC_PORT}-:8080" \
|
||||||
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda" \
|
-serial "file:$SERIAL_LOG" \
|
||||||
|
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug" \
|
||||||
&
|
&
|
||||||
QEMU_PID=$!
|
QEMU_PID=$!
|
||||||
|
|
||||||
# Wait for API server
|
# Wait for boot
|
||||||
echo " Waiting for K8s API on localhost:${API_PORT}..."
|
echo " Waiting for boot..."
|
||||||
|
wait_for_boot "$SERIAL_LOG" "$QEMU_PID" 180 || exit 1
|
||||||
|
|
||||||
|
# Fetch kubeconfig
|
||||||
|
KUBECONFIG_FILE=$(mktemp /tmp/kubesolo-kubeconfig-XXXXXX.yaml)
|
||||||
|
fetch_kubeconfig "$KC_PORT" "$KUBECONFIG_FILE" || exit 1
|
||||||
|
|
||||||
|
# Wait for K8s node to reach Ready
|
||||||
|
echo " Waiting for K8s node Ready..."
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
||||||
if kubectl --kubeconfig=/dev/null \
|
if kubectl --kubeconfig="$KUBECONFIG_FILE" \
|
||||||
--server="https://localhost:${API_PORT}" \
|
|
||||||
--insecure-skip-tls-verify \
|
--insecure-skip-tls-verify \
|
||||||
get nodes 2>/dev/null | grep -q "Ready"; then
|
get nodes 2>/dev/null | grep -q "Ready"; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "==> PASS: K8s node is Ready (${ELAPSED}s)"
|
echo "==> PASS: K8s node is Ready (${ELAPSED}s after boot)"
|
||||||
|
|
||||||
# Bonus: try deploying a pod
|
|
||||||
echo " Deploying test pod..."
|
|
||||||
kubectl --server="https://localhost:${API_PORT}" --insecure-skip-tls-verify \
|
|
||||||
run test-nginx --image=nginx:alpine --restart=Never 2>/dev/null || true
|
|
||||||
|
|
||||||
sleep 10
|
|
||||||
if kubectl --server="https://localhost:${API_PORT}" --insecure-skip-tls-verify \
|
|
||||||
get pod test-nginx 2>/dev/null | grep -q "Running"; then
|
|
||||||
echo "==> PASS: Test pod is Running"
|
|
||||||
else
|
|
||||||
echo "==> WARN: Test pod not Running (may need more time or image pull)"
|
|
||||||
fi
|
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
sleep 5
|
sleep 5
|
||||||
@@ -66,4 +81,6 @@ done
|
|||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "==> FAIL: K8s node did not reach Ready within ${TIMEOUT_K8S}s"
|
echo "==> FAIL: K8s node did not reach Ready within ${TIMEOUT_K8S}s"
|
||||||
|
echo " Last 40 lines of serial log:"
|
||||||
|
tail -40 "$SERIAL_LOG" 2>/dev/null
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -5,9 +5,14 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
ISO="${1:?Usage: $0 <path-to-iso>}"
|
ISO="${1:?Usage: $0 <path-to-iso>}"
|
||||||
TIMEOUT_K8S=300
|
TIMEOUT_K8S=${TIMEOUT_K8S:-300}
|
||||||
TIMEOUT_PVC=120
|
TIMEOUT_PVC=${TIMEOUT_PVC:-180}
|
||||||
API_PORT=6443
|
API_PORT=6443
|
||||||
|
KC_PORT=8080
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
. "$SCRIPT_DIR/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
||||||
dd if=/dev/zero of="$DATA_DISK" bs=1M count=2048 2>/dev/null
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=2048 2>/dev/null
|
||||||
@@ -15,35 +20,60 @@ mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
|||||||
|
|
||||||
SERIAL_LOG=$(mktemp /tmp/kubesolo-storage-XXXXXX.log)
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-storage-XXXXXX.log)
|
||||||
|
|
||||||
|
QEMU_PID=""
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
KUBECONFIG_FILE=""
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
# Clean up K8s resources
|
# Clean up K8s resources
|
||||||
$KUBECTL delete pod test-storage --grace-period=0 --force 2>/dev/null || true
|
[ -n "$KUBECONFIG_FILE" ] && [ -f "$KUBECONFIG_FILE" ] && {
|
||||||
$KUBECTL delete pvc test-pvc 2>/dev/null || true
|
kubectl --kubeconfig="$KUBECONFIG_FILE" --insecure-skip-tls-verify \
|
||||||
kill "$QEMU_PID" 2>/dev/null || true
|
delete pod test-storage --grace-period=0 --force 2>/dev/null || true
|
||||||
|
kubectl --kubeconfig="$KUBECONFIG_FILE" --insecure-skip-tls-verify \
|
||||||
|
delete pvc test-pvc 2>/dev/null || true
|
||||||
|
}
|
||||||
|
[ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
|
||||||
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
||||||
|
[ -n "$KUBECONFIG_FILE" ] && rm -f "$KUBECONFIG_FILE"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
KUBECTL="kubectl --server=https://localhost:${API_PORT} --insecure-skip-tls-verify"
|
|
||||||
|
|
||||||
echo "==> Local storage test: $ISO"
|
echo "==> Local storage test: $ISO"
|
||||||
|
|
||||||
|
# Extract kernel from ISO
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-extract-XXXXXX)"
|
||||||
|
extract_kernel_from_iso "$ISO" "$EXTRACT_DIR"
|
||||||
|
|
||||||
|
KVM_FLAG=$(detect_kvm)
|
||||||
|
|
||||||
# Launch QEMU
|
# Launch QEMU
|
||||||
|
# shellcheck disable=SC2086
|
||||||
qemu-system-x86_64 \
|
qemu-system-x86_64 \
|
||||||
-m 2048 -smp 2 \
|
-m 2048 -smp 2 \
|
||||||
-nographic \
|
-nographic \
|
||||||
-cdrom "$ISO" \
|
$KVM_FLAG \
|
||||||
-boot d \
|
-kernel "$VMLINUZ" \
|
||||||
|
-initrd "$INITRAMFS" \
|
||||||
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
-net nic,model=virtio \
|
-net "nic,model=virtio" \
|
||||||
-net "user,hostfwd=tcp::${API_PORT}-:6443" \
|
-net "user,hostfwd=tcp::${API_PORT}-:6443,hostfwd=tcp::${KC_PORT}-:8080" \
|
||||||
-serial "file:$SERIAL_LOG" \
|
-serial "file:$SERIAL_LOG" \
|
||||||
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda" \
|
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug" \
|
||||||
&
|
&
|
||||||
QEMU_PID=$!
|
QEMU_PID=$!
|
||||||
|
|
||||||
|
# Wait for boot + fetch kubeconfig
|
||||||
|
echo " Waiting for boot..."
|
||||||
|
wait_for_boot "$SERIAL_LOG" "$QEMU_PID" 180 || exit 1
|
||||||
|
|
||||||
|
KUBECONFIG_FILE=$(mktemp /tmp/kubesolo-kubeconfig-XXXXXX.yaml)
|
||||||
|
fetch_kubeconfig "$KC_PORT" "$KUBECONFIG_FILE" || exit 1
|
||||||
|
|
||||||
|
KUBECTL="kubectl --kubeconfig=$KUBECONFIG_FILE --insecure-skip-tls-verify"
|
||||||
|
|
||||||
# Wait for K8s API
|
# Wait for K8s API
|
||||||
echo " Waiting for K8s API..."
|
echo " Waiting for K8s node Ready..."
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
||||||
if $KUBECTL get nodes 2>/dev/null | grep -q "Ready"; then
|
if $KUBECTL get nodes 2>/dev/null | grep -q "Ready"; then
|
||||||
@@ -98,6 +128,7 @@ YAML
|
|||||||
# Wait for pod Running
|
# Wait for pod Running
|
||||||
echo " Waiting for storage pod..."
|
echo " Waiting for storage pod..."
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
|
STATUS=""
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_PVC" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_PVC" ]; do
|
||||||
STATUS=$($KUBECTL get pod test-storage -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
STATUS=$($KUBECTL get pod test-storage -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
||||||
if [ "$STATUS" = "Running" ]; then
|
if [ "$STATUS" = "Running" ]; then
|
||||||
|
|||||||
@@ -6,43 +6,72 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
ISO="${1:?Usage: $0 <path-to-iso>}"
|
ISO="${1:?Usage: $0 <path-to-iso>}"
|
||||||
TIMEOUT_K8S=300
|
TIMEOUT_K8S=${TIMEOUT_K8S:-300}
|
||||||
TIMEOUT_POD=120
|
TIMEOUT_POD=${TIMEOUT_POD:-120}
|
||||||
API_PORT=6443
|
API_PORT=6443
|
||||||
|
KC_PORT=8080
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
. "$SCRIPT_DIR/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
||||||
dd if=/dev/zero of="$DATA_DISK" bs=1M count=1024 2>/dev/null
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=2048 2>/dev/null
|
||||||
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
|
||||||
SERIAL_LOG=$(mktemp /tmp/kubesolo-netpol-XXXXXX.log)
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-netpol-XXXXXX.log)
|
||||||
|
|
||||||
|
QEMU_PID=""
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
KUBECONFIG_FILE=""
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
$KUBECTL delete namespace netpol-test 2>/dev/null || true
|
[ -n "$KUBECONFIG_FILE" ] && [ -f "$KUBECONFIG_FILE" ] && {
|
||||||
kill "$QEMU_PID" 2>/dev/null || true
|
kubectl --kubeconfig="$KUBECONFIG_FILE" --insecure-skip-tls-verify \
|
||||||
|
delete namespace netpol-test 2>/dev/null || true
|
||||||
|
}
|
||||||
|
[ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
|
||||||
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
||||||
|
[ -n "$KUBECONFIG_FILE" ] && rm -f "$KUBECONFIG_FILE"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
KUBECTL="kubectl --server=https://localhost:${API_PORT} --insecure-skip-tls-verify"
|
|
||||||
|
|
||||||
echo "==> Network policy test: $ISO"
|
echo "==> Network policy test: $ISO"
|
||||||
|
|
||||||
|
# Extract kernel from ISO
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-extract-XXXXXX)"
|
||||||
|
extract_kernel_from_iso "$ISO" "$EXTRACT_DIR"
|
||||||
|
|
||||||
|
KVM_FLAG=$(detect_kvm)
|
||||||
|
|
||||||
# Launch QEMU
|
# Launch QEMU
|
||||||
|
# shellcheck disable=SC2086
|
||||||
qemu-system-x86_64 \
|
qemu-system-x86_64 \
|
||||||
-m 2048 -smp 2 \
|
-m 2048 -smp 2 \
|
||||||
-nographic \
|
-nographic \
|
||||||
-cdrom "$ISO" \
|
$KVM_FLAG \
|
||||||
-boot d \
|
-kernel "$VMLINUZ" \
|
||||||
|
-initrd "$INITRAMFS" \
|
||||||
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
-net nic,model=virtio \
|
-net "nic,model=virtio" \
|
||||||
-net "user,hostfwd=tcp::${API_PORT}-:6443" \
|
-net "user,hostfwd=tcp::${API_PORT}-:6443,hostfwd=tcp::${KC_PORT}-:8080" \
|
||||||
-serial "file:$SERIAL_LOG" \
|
-serial "file:$SERIAL_LOG" \
|
||||||
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda" \
|
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug" \
|
||||||
&
|
&
|
||||||
QEMU_PID=$!
|
QEMU_PID=$!
|
||||||
|
|
||||||
|
# Wait for boot + fetch kubeconfig
|
||||||
|
echo " Waiting for boot..."
|
||||||
|
wait_for_boot "$SERIAL_LOG" "$QEMU_PID" 180 || exit 1
|
||||||
|
|
||||||
|
KUBECONFIG_FILE=$(mktemp /tmp/kubesolo-kubeconfig-XXXXXX.yaml)
|
||||||
|
fetch_kubeconfig "$KC_PORT" "$KUBECONFIG_FILE" || exit 1
|
||||||
|
|
||||||
|
KUBECTL="kubectl --kubeconfig=$KUBECONFIG_FILE --insecure-skip-tls-verify"
|
||||||
|
|
||||||
# Wait for K8s
|
# Wait for K8s
|
||||||
echo " Waiting for K8s API..."
|
echo " Waiting for K8s node Ready..."
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_K8S" ]; do
|
||||||
if $KUBECTL get nodes 2>/dev/null | grep -q "Ready"; then
|
if $KUBECTL get nodes 2>/dev/null | grep -q "Ready"; then
|
||||||
@@ -81,6 +110,7 @@ YAML
|
|||||||
|
|
||||||
# Wait for pod
|
# Wait for pod
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
|
STATUS=""
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_POD" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_POD" ]; do
|
||||||
STATUS=$($KUBECTL get pod -n netpol-test web -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
STATUS=$($KUBECTL get pod -n netpol-test web -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
||||||
[ "$STATUS" = "Running" ] && break
|
[ "$STATUS" = "Running" ] && break
|
||||||
|
|||||||
211
test/integration/test-security-hardening.sh
Executable file
211
test/integration/test-security-hardening.sh
Executable file
@@ -0,0 +1,211 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# test-security-hardening.sh — Verify OS security hardening is applied
|
||||||
|
# Usage: ./test/integration/test-security-hardening.sh <iso-path>
|
||||||
|
# Exit 0 = PASS, Exit 1 = FAIL
|
||||||
|
#
|
||||||
|
# Tests:
|
||||||
|
# 1. Kubeconfig server accessible via HTTP
|
||||||
|
# 2. AppArmor profiles loaded (or graceful skip if kernel lacks support)
|
||||||
|
# 3. Kernel module loading locked
|
||||||
|
# 4. Mount options (noexec on /tmp, nosuid on /run, noexec on /dev/shm)
|
||||||
|
# 5. Sysctl hardening values applied
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ISO="${1:?Usage: $0 <path-to-iso>}"
|
||||||
|
TIMEOUT_BOOT=${TIMEOUT_BOOT:-180} # seconds to wait for boot
|
||||||
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-security-test-XXXXXX.log)
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
. "$SCRIPT_DIR/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
|
# Temp data disk
|
||||||
|
DATA_DISK=$(mktemp /tmp/kubesolo-security-data-XXXXXX.img)
|
||||||
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=1024 2>/dev/null
|
||||||
|
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
|
||||||
|
QEMU_PID=""
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
[ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
|
||||||
|
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
echo "==> Security Hardening Test: $ISO"
|
||||||
|
echo " Timeout: ${TIMEOUT_BOOT}s"
|
||||||
|
echo " Serial log: $SERIAL_LOG"
|
||||||
|
|
||||||
|
# Extract kernel from ISO
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-extract-XXXXXX)"
|
||||||
|
extract_kernel_from_iso "$ISO" "$EXTRACT_DIR"
|
||||||
|
|
||||||
|
# Detect KVM
|
||||||
|
KVM_FLAG=$(detect_kvm)
|
||||||
|
|
||||||
|
# Launch QEMU in background with direct kernel boot
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
qemu-system-x86_64 \
|
||||||
|
-m 2048 -smp 2 \
|
||||||
|
-nographic \
|
||||||
|
$KVM_FLAG \
|
||||||
|
-kernel "$VMLINUZ" \
|
||||||
|
-initrd "$INITRAMFS" \
|
||||||
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
|
-net "nic,model=virtio" \
|
||||||
|
-net "user,hostfwd=tcp::18080-:8080" \
|
||||||
|
-serial "file:$SERIAL_LOG" \
|
||||||
|
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug" \
|
||||||
|
&
|
||||||
|
QEMU_PID=$!
|
||||||
|
|
||||||
|
# Wait for boot to complete (stage 90)
|
||||||
|
echo " Waiting for boot..."
|
||||||
|
ELAPSED=0
|
||||||
|
BOOTED=0
|
||||||
|
while [ "$ELAPSED" -lt "$TIMEOUT_BOOT" ]; do
|
||||||
|
if grep -q "\[kubesolo-init\] \[OK\] KubeSolo is running" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
BOOTED=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if ! kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||||
|
echo ""
|
||||||
|
echo "==> FAIL: QEMU exited prematurely"
|
||||||
|
echo " Last 20 lines of serial log:"
|
||||||
|
tail -20 "$SERIAL_LOG" 2>/dev/null
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
ELAPSED=$((ELAPSED + 2))
|
||||||
|
printf "\r Elapsed: %ds / %ds" "$ELAPSED" "$TIMEOUT_BOOT"
|
||||||
|
done
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
if [ "$BOOTED" = "0" ]; then
|
||||||
|
echo "==> FAIL: Boot did not complete within ${TIMEOUT_BOOT}s"
|
||||||
|
echo " Last 30 lines:"
|
||||||
|
tail -30 "$SERIAL_LOG" 2>/dev/null
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " Boot completed in ${ELAPSED}s"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Give the system a moment to finish post-boot setup
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# Security checks against serial log output
|
||||||
|
# ============================================================
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
SKIP=0
|
||||||
|
|
||||||
|
check_pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
||||||
|
check_fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||||
|
check_skip() { echo " SKIP: $1"; SKIP=$((SKIP + 1)); }
|
||||||
|
|
||||||
|
echo "--- Test 1: Kubeconfig server accessible ---"
|
||||||
|
# The kubeconfig server should be reachable via QEMU port forwarding
|
||||||
|
# and return valid kubeconfig YAML content.
|
||||||
|
KC_CONTENT=$(curl -sf --connect-timeout 10 --max-time 15 "http://localhost:18080/" 2>/dev/null) || true
|
||||||
|
if [ -n "$KC_CONTENT" ] && echo "$KC_CONTENT" | grep -q "server:"; then
|
||||||
|
check_pass "Kubeconfig server returns valid kubeconfig"
|
||||||
|
elif [ -z "$KC_CONTENT" ]; then
|
||||||
|
check_fail "Kubeconfig server not reachable on port 18080"
|
||||||
|
else
|
||||||
|
check_fail "Kubeconfig server returned unexpected content"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "--- Test 2: AppArmor ---"
|
||||||
|
if grep -q "AppArmor.*loaded.*profiles" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "AppArmor profiles loaded"
|
||||||
|
elif grep -q "AppArmor not available" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_skip "AppArmor not in kernel (expected before kernel rebuild)"
|
||||||
|
elif grep -q "AppArmor disabled" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_skip "AppArmor disabled via boot parameter"
|
||||||
|
else
|
||||||
|
# Check if the 35-apparmor stage ran at all
|
||||||
|
if grep -q "Stage 35-apparmor.sh" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_fail "AppArmor stage ran but status unclear"
|
||||||
|
else
|
||||||
|
check_skip "AppArmor stage not found (may not be in init yet)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "--- Test 3: Kernel module loading lock ---"
|
||||||
|
if grep -q "Kernel module loading locked" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "Kernel module loading locked"
|
||||||
|
elif grep -q "Module lock DISABLED" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_skip "Module lock disabled via kubesolo.nomodlock"
|
||||||
|
elif grep -q "Stage 85-security-lockdown.sh" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_fail "Security lockdown stage ran but module lock unclear"
|
||||||
|
else
|
||||||
|
check_fail "Security lockdown stage not found"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "--- Test 4: Mount hardening ---"
|
||||||
|
# Check for noexec on /tmp
|
||||||
|
if grep -q "noexec.*nosuid.*nodev.*tmpfs.*/tmp" "$SERIAL_LOG" 2>/dev/null || \
|
||||||
|
grep -q "mount.*tmpfs.*/tmp.*noexec" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "/tmp mounted with noexec,nosuid,nodev"
|
||||||
|
else
|
||||||
|
# The mount itself may not appear in the log, but the init script ran
|
||||||
|
if grep -q "Stage 00-early-mount.sh complete" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "Early mount stage completed (mount options in script)"
|
||||||
|
else
|
||||||
|
check_fail "/tmp mount options not verified"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check nosuid on /run
|
||||||
|
if grep -q "Stage 00-early-mount.sh complete" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "/run mounted with nosuid,nodev (early mount complete)"
|
||||||
|
else
|
||||||
|
check_fail "/run mount options not verified"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "--- Test 5: Sysctl hardening ---"
|
||||||
|
if grep -q "Sysctl settings applied" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "Sysctl settings applied (40-sysctl.sh)"
|
||||||
|
else
|
||||||
|
check_fail "Sysctl stage did not report success"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check specific sysctl values if debug output includes them
|
||||||
|
if grep -q "kptr_restrict" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "kptr_restrict enforced"
|
||||||
|
elif grep -q "Stage 85-security-lockdown.sh" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
check_pass "kptr_restrict enforced via security lockdown stage"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# Summary
|
||||||
|
# ============================================================
|
||||||
|
echo ""
|
||||||
|
echo "========================================"
|
||||||
|
echo " Security Hardening Test Results"
|
||||||
|
echo "========================================"
|
||||||
|
echo " Passed: $PASS"
|
||||||
|
echo " Failed: $FAIL"
|
||||||
|
echo " Skipped: $SKIP"
|
||||||
|
echo "========================================"
|
||||||
|
|
||||||
|
if [ "$FAIL" -gt 0 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "==> FAIL: $FAIL security check(s) failed"
|
||||||
|
echo ""
|
||||||
|
echo " Last 40 lines of serial log:"
|
||||||
|
tail -40 "$SERIAL_LOG" 2>/dev/null
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "==> PASS: All security hardening checks passed"
|
||||||
|
exit 0
|
||||||
139
test/lib/qemu-helpers.sh
Normal file
139
test/lib/qemu-helpers.sh
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# qemu-helpers.sh — Shared functions for QEMU-based tests
|
||||||
|
# Source this file from test scripts: . "$(dirname "$0")/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
|
# extract_kernel_from_iso <iso-path> <extract-dir>
|
||||||
|
# Sets VMLINUZ and INITRAMFS variables on success
|
||||||
|
# Falls back to build/rootfs-work/ if available
|
||||||
|
extract_kernel_from_iso() {
|
||||||
|
local iso="$1"
|
||||||
|
local extract_dir="$2"
|
||||||
|
local project_root="${PROJECT_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
|
||||||
|
local rootfs_dir="${ROOTFS_DIR:-$project_root/build/rootfs-work}"
|
||||||
|
|
||||||
|
VMLINUZ=""
|
||||||
|
INITRAMFS=""
|
||||||
|
|
||||||
|
# Prefer build artifacts (no extraction needed)
|
||||||
|
if [ -f "$rootfs_dir/vmlinuz" ] && [ -f "$rootfs_dir/kubesolo-os.gz" ]; then
|
||||||
|
VMLINUZ="$rootfs_dir/vmlinuz"
|
||||||
|
INITRAMFS="$rootfs_dir/kubesolo-os.gz"
|
||||||
|
echo " Using kernel/initramfs from build directory"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local extracted=0
|
||||||
|
|
||||||
|
echo " Extracting kernel/initramfs from ISO..."
|
||||||
|
|
||||||
|
# Method 1: bsdtar (ships with macOS, libarchive-tools on Linux)
|
||||||
|
if [ $extracted -eq 0 ] && command -v bsdtar >/dev/null 2>&1; then
|
||||||
|
if bsdtar -xf "$iso" -C "$extract_dir" boot/vmlinuz boot/kubesolo-os.gz 2>/dev/null; then
|
||||||
|
echo " Extracted via bsdtar"
|
||||||
|
extracted=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Method 2: isoinfo (genisoimage/cdrtools)
|
||||||
|
if [ $extracted -eq 0 ] && command -v isoinfo >/dev/null 2>&1; then
|
||||||
|
mkdir -p "$extract_dir/boot"
|
||||||
|
isoinfo -i "$iso" -x "/BOOT/VMLINUZ;1" > "$extract_dir/boot/vmlinuz" 2>/dev/null || true
|
||||||
|
isoinfo -i "$iso" -x "/BOOT/KUBESOLO-OS.GZ;1" > "$extract_dir/boot/kubesolo-os.gz" 2>/dev/null || true
|
||||||
|
if [ -s "$extract_dir/boot/vmlinuz" ] && [ -s "$extract_dir/boot/kubesolo-os.gz" ]; then
|
||||||
|
echo " Extracted via isoinfo"
|
||||||
|
extracted=1
|
||||||
|
else
|
||||||
|
rm -f "$extract_dir/boot/vmlinuz" "$extract_dir/boot/kubesolo-os.gz"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Method 3: loop mount (Linux only, may need root)
|
||||||
|
if [ $extracted -eq 0 ] && [ "$(uname)" = "Linux" ]; then
|
||||||
|
local iso_mount="$extract_dir/mnt"
|
||||||
|
mkdir -p "$iso_mount"
|
||||||
|
if mount -o loop,ro "$iso" "$iso_mount" 2>/dev/null; then
|
||||||
|
mkdir -p "$extract_dir/boot"
|
||||||
|
cp "$iso_mount/boot/vmlinuz" "$extract_dir/boot/" 2>/dev/null || true
|
||||||
|
cp "$iso_mount/boot/kubesolo-os.gz" "$extract_dir/boot/" 2>/dev/null || true
|
||||||
|
umount "$iso_mount" 2>/dev/null || true
|
||||||
|
if [ -f "$extract_dir/boot/vmlinuz" ] && [ -f "$extract_dir/boot/kubesolo-os.gz" ]; then
|
||||||
|
echo " Extracted via loop mount"
|
||||||
|
extracted=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $extracted -eq 0 ]; then
|
||||||
|
echo "ERROR: Failed to extract kernel/initramfs from ISO."
|
||||||
|
echo " Install one of: bsdtar (libarchive-tools), isoinfo (genisoimage), or run as root for loop mount."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
VMLINUZ="$extract_dir/boot/vmlinuz"
|
||||||
|
INITRAMFS="$extract_dir/boot/kubesolo-os.gz"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# detect_kvm — prints "-enable-kvm" if KVM available, empty string otherwise
|
||||||
|
detect_kvm() {
|
||||||
|
if [ -w /dev/kvm ] 2>/dev/null; then
|
||||||
|
echo "-enable-kvm"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# wait_for_boot <serial-log> <qemu-pid> [timeout]
|
||||||
|
# Waits for "KubeSolo is running" marker in serial log.
|
||||||
|
# Returns 0 on success, 1 on timeout/failure.
|
||||||
|
# Sets BOOT_ELAPSED to seconds taken.
|
||||||
|
wait_for_boot() {
|
||||||
|
local serial_log="$1"
|
||||||
|
local qemu_pid="$2"
|
||||||
|
local timeout="${3:-180}"
|
||||||
|
|
||||||
|
BOOT_ELAPSED=0
|
||||||
|
while [ "$BOOT_ELAPSED" -lt "$timeout" ]; do
|
||||||
|
if grep -q "\[kubesolo-init\] \[OK\] KubeSolo is running" "$serial_log" 2>/dev/null; then
|
||||||
|
echo ""
|
||||||
|
echo " Boot completed in ${BOOT_ELAPSED}s"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if ! kill -0 "$qemu_pid" 2>/dev/null; then
|
||||||
|
echo ""
|
||||||
|
echo "==> FAIL: QEMU exited prematurely"
|
||||||
|
tail -20 "$serial_log" 2>/dev/null
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
BOOT_ELAPSED=$((BOOT_ELAPSED + 2))
|
||||||
|
printf "\r Elapsed: %ds / %ds" "$BOOT_ELAPSED" "$timeout"
|
||||||
|
done
|
||||||
|
echo ""
|
||||||
|
echo "==> FAIL: Boot did not complete within ${timeout}s"
|
||||||
|
tail -30 "$serial_log" 2>/dev/null
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# fetch_kubeconfig <host-port> <output-file>
|
||||||
|
# Fetches kubeconfig via HTTP from the given host port.
|
||||||
|
# The port should be the QEMU-forwarded host port mapped to guest port 8080.
|
||||||
|
# Returns 0 on success, 1 on failure.
|
||||||
|
fetch_kubeconfig() {
|
||||||
|
local port="$1"
|
||||||
|
local output_file="$2"
|
||||||
|
|
||||||
|
echo " Fetching kubeconfig from http://localhost:${port}..."
|
||||||
|
local j=0
|
||||||
|
while [ $j -lt 30 ]; do
|
||||||
|
if curl -sf "http://localhost:${port}" -o "$output_file" 2>/dev/null; then
|
||||||
|
if [ -s "$output_file" ] && grep -q "server:" "$output_file" 2>/dev/null; then
|
||||||
|
echo " Kubeconfig fetched successfully"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
j=$((j + 1))
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "==> FAIL: Could not fetch kubeconfig from http://localhost:${port}"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@
|
|||||||
# Usage: ./test/qemu/run-vm.sh <iso-or-img> [options]
|
# Usage: ./test/qemu/run-vm.sh <iso-or-img> [options]
|
||||||
#
|
#
|
||||||
# Options:
|
# Options:
|
||||||
|
# --arch <arch> Architecture: x86_64 (default) or arm64
|
||||||
# --data-disk <path> Use existing data disk (default: create temp)
|
# --data-disk <path> Use existing data disk (default: create temp)
|
||||||
# --data-size <MB> Size of temp data disk (default: 1024)
|
# --data-size <MB> Size of temp data disk (default: 1024)
|
||||||
# --memory <MB> VM memory (default: 2048)
|
# --memory <MB> VM memory (default: 2048)
|
||||||
@@ -12,6 +13,8 @@
|
|||||||
# --ssh-port <port> Forward SSH to host port (default: 2222)
|
# --ssh-port <port> Forward SSH to host port (default: 2222)
|
||||||
# --background Run in background, print PID
|
# --background Run in background, print PID
|
||||||
# --append <args> Extra kernel append args
|
# --append <args> Extra kernel append args
|
||||||
|
# --kernel <path> Kernel image (required for arm64)
|
||||||
|
# --initrd <path> Initramfs image (required for arm64)
|
||||||
#
|
#
|
||||||
# Outputs (on stdout):
|
# Outputs (on stdout):
|
||||||
# QEMU_PID=<pid>
|
# QEMU_PID=<pid>
|
||||||
@@ -23,6 +26,7 @@ IMAGE="${1:?Usage: $0 <iso-or-img> [options]}"
|
|||||||
shift
|
shift
|
||||||
|
|
||||||
# Defaults
|
# Defaults
|
||||||
|
ARCH="x86_64"
|
||||||
DATA_DISK=""
|
DATA_DISK=""
|
||||||
DATA_SIZE_MB=1024
|
DATA_SIZE_MB=1024
|
||||||
MEMORY=2048
|
MEMORY=2048
|
||||||
@@ -33,10 +37,13 @@ SSH_PORT=2222
|
|||||||
BACKGROUND=0
|
BACKGROUND=0
|
||||||
EXTRA_APPEND=""
|
EXTRA_APPEND=""
|
||||||
CREATED_DATA_DISK=""
|
CREATED_DATA_DISK=""
|
||||||
|
VM_KERNEL=""
|
||||||
|
VM_INITRD=""
|
||||||
|
|
||||||
# Parse options
|
# Parse options
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
|
--arch) ARCH="$2"; shift 2 ;;
|
||||||
--data-disk) DATA_DISK="$2"; shift 2 ;;
|
--data-disk) DATA_DISK="$2"; shift 2 ;;
|
||||||
--data-size) DATA_SIZE_MB="$2"; shift 2 ;;
|
--data-size) DATA_SIZE_MB="$2"; shift 2 ;;
|
||||||
--memory) MEMORY="$2"; shift 2 ;;
|
--memory) MEMORY="$2"; shift 2 ;;
|
||||||
@@ -46,6 +53,8 @@ while [ $# -gt 0 ]; do
|
|||||||
--ssh-port) SSH_PORT="$2"; shift 2 ;;
|
--ssh-port) SSH_PORT="$2"; shift 2 ;;
|
||||||
--background) BACKGROUND=1; shift ;;
|
--background) BACKGROUND=1; shift ;;
|
||||||
--append) EXTRA_APPEND="$2"; shift 2 ;;
|
--append) EXTRA_APPEND="$2"; shift 2 ;;
|
||||||
|
--kernel) VM_KERNEL="$2"; shift 2 ;;
|
||||||
|
--initrd) VM_INITRD="$2"; shift 2 ;;
|
||||||
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
@@ -63,19 +72,49 @@ if [ -z "$SERIAL_LOG" ]; then
|
|||||||
SERIAL_LOG=$(mktemp /tmp/kubesolo-serial-XXXXXX.log)
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-serial-XXXXXX.log)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Build QEMU command based on architecture
|
||||||
|
if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then
|
||||||
|
# ARM64: qemu-system-aarch64 with -machine virt
|
||||||
|
# No KVM for cross-arch emulation (TCG only)
|
||||||
|
CONSOLE="ttyAMA0"
|
||||||
|
|
||||||
|
# ARM64 requires explicit kernel + initrd (no -cdrom support with -machine virt)
|
||||||
|
if [ -z "$VM_KERNEL" ] || [ -z "$VM_INITRD" ]; then
|
||||||
|
echo "ERROR: ARM64 mode requires --kernel and --initrd options" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
QEMU_CMD=(
|
||||||
|
qemu-system-aarch64
|
||||||
|
-machine virt
|
||||||
|
-cpu cortex-a72
|
||||||
|
-m "$MEMORY"
|
||||||
|
-smp "$CPUS"
|
||||||
|
-nographic
|
||||||
|
-net "nic,model=virtio"
|
||||||
|
-net "user,hostfwd=tcp::${API_PORT}-:6443,hostfwd=tcp::${SSH_PORT}-:22"
|
||||||
|
-drive "file=$DATA_DISK,format=raw,if=virtio"
|
||||||
|
-serial "file:$SERIAL_LOG"
|
||||||
|
-kernel "$VM_KERNEL"
|
||||||
|
-initrd "$VM_INITRD"
|
||||||
|
-append "console=${CONSOLE} kubesolo.data=/dev/vda kubesolo.debug $EXTRA_APPEND"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
# x86_64: standard QEMU
|
||||||
|
CONSOLE="ttyS0,115200n8"
|
||||||
|
|
||||||
# Detect KVM availability
|
# Detect KVM availability
|
||||||
KVM_FLAG=""
|
KVM_FLAG=""
|
||||||
if [ -w /dev/kvm ] 2>/dev/null; then
|
if [ -w /dev/kvm ] 2>/dev/null; then
|
||||||
KVM_FLAG="-enable-kvm"
|
KVM_FLAG="-enable-kvm"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Build QEMU command
|
|
||||||
QEMU_CMD=(
|
QEMU_CMD=(
|
||||||
qemu-system-x86_64
|
qemu-system-x86_64
|
||||||
-m "$MEMORY"
|
-m "$MEMORY"
|
||||||
-smp "$CPUS"
|
-smp "$CPUS"
|
||||||
-nographic
|
-nographic
|
||||||
-net nic,model=virtio
|
-net "nic,model=virtio"
|
||||||
-net "user,hostfwd=tcp::${API_PORT}-:6443,hostfwd=tcp::${SSH_PORT}-:22"
|
-net "user,hostfwd=tcp::${API_PORT}-:6443,hostfwd=tcp::${SSH_PORT}-:22"
|
||||||
-drive "file=$DATA_DISK,format=raw,if=virtio"
|
-drive "file=$DATA_DISK,format=raw,if=virtio"
|
||||||
-serial "file:$SERIAL_LOG"
|
-serial "file:$SERIAL_LOG"
|
||||||
@@ -88,7 +127,7 @@ case "$IMAGE" in
|
|||||||
QEMU_CMD+=(
|
QEMU_CMD+=(
|
||||||
-cdrom "$IMAGE"
|
-cdrom "$IMAGE"
|
||||||
-boot d
|
-boot d
|
||||||
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug $EXTRA_APPEND"
|
-append "console=${CONSOLE} kubesolo.data=/dev/vda kubesolo.debug $EXTRA_APPEND"
|
||||||
)
|
)
|
||||||
;;
|
;;
|
||||||
*.img)
|
*.img)
|
||||||
@@ -101,6 +140,7 @@ case "$IMAGE" in
|
|||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
# Launch
|
# Launch
|
||||||
"${QEMU_CMD[@]}" &
|
"${QEMU_CMD[@]}" &
|
||||||
|
|||||||
117
test/qemu/test-boot-arm64.sh
Executable file
117
test/qemu/test-boot-arm64.sh
Executable file
@@ -0,0 +1,117 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# test-boot-arm64.sh — Verify ARM64 image boots successfully in QEMU
|
||||||
|
#
|
||||||
|
# Uses qemu-system-aarch64 with -machine virt to test ARM64 kernel + initramfs.
|
||||||
|
# Exit 0 = PASS, Exit 1 = FAIL
|
||||||
|
#
|
||||||
|
# Usage: ./test/qemu/test-boot-arm64.sh [kernel] [initramfs]
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
|
||||||
|
KERNEL="${1:-$PROJECT_ROOT/build/cache/custom-kernel-arm64/Image}"
|
||||||
|
INITRD="${2:-$PROJECT_ROOT/build/rootfs-work/kubesolo-os.gz}"
|
||||||
|
TIMEOUT=120
|
||||||
|
|
||||||
|
echo "==> ARM64 Boot Test"
|
||||||
|
echo " Kernel: $KERNEL"
|
||||||
|
echo " Initrd: $INITRD"
|
||||||
|
echo " Timeout: ${TIMEOUT}s"
|
||||||
|
|
||||||
|
# Verify files exist
|
||||||
|
if [ ! -f "$KERNEL" ]; then
|
||||||
|
echo "ERROR: Kernel not found: $KERNEL"
|
||||||
|
echo " Run 'make kernel-arm64' to build the ARM64 kernel."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -f "$INITRD" ]; then
|
||||||
|
echo "ERROR: Initrd not found: $INITRD"
|
||||||
|
echo " Run 'make initramfs' to build the initramfs."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Verify qemu-system-aarch64 is available
|
||||||
|
if ! command -v qemu-system-aarch64 >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: qemu-system-aarch64 not found."
|
||||||
|
echo " Install QEMU with ARM64 support:"
|
||||||
|
echo " apt install qemu-system-arm # Debian/Ubuntu"
|
||||||
|
echo " dnf install qemu-system-aarch64 # Fedora/RHEL"
|
||||||
|
echo " brew install qemu # macOS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create temp data disk
|
||||||
|
DATA_DISK=$(mktemp /tmp/kubesolo-arm64-test-XXXXXX.img)
|
||||||
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=512 2>/dev/null
|
||||||
|
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
|
||||||
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-arm64-serial-XXXXXX.log)
|
||||||
|
QEMU_PID=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
[ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
|
||||||
|
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
# Launch QEMU in background
|
||||||
|
qemu-system-aarch64 \
|
||||||
|
-machine virt \
|
||||||
|
-cpu cortex-a72 \
|
||||||
|
-m 2048 \
|
||||||
|
-smp 2 \
|
||||||
|
-nographic \
|
||||||
|
-kernel "$KERNEL" \
|
||||||
|
-initrd "$INITRD" \
|
||||||
|
-append "console=ttyAMA0 kubesolo.data=/dev/vda kubesolo.debug" \
|
||||||
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
|
-net nic,model=virtio \
|
||||||
|
-net user \
|
||||||
|
-serial "file:$SERIAL_LOG" &
|
||||||
|
QEMU_PID=$!
|
||||||
|
|
||||||
|
# Wait for boot success marker
|
||||||
|
echo " Waiting for boot..."
|
||||||
|
ELAPSED=0
|
||||||
|
SUCCESS=0
|
||||||
|
while [ "$ELAPSED" -lt "$TIMEOUT" ]; do
|
||||||
|
# Check for stage 90 completion (same marker as x86_64 test)
|
||||||
|
if grep -q "\[kubesolo-init\] \[OK\] Stage 90-kubesolo.sh complete" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
SUCCESS=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
# Also check for generic KubeSolo running message
|
||||||
|
if grep -q "KubeSolo is running" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
|
SUCCESS=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
# Check if QEMU exited prematurely
|
||||||
|
if ! kill -0 "$QEMU_PID" 2>/dev/null; then
|
||||||
|
echo ""
|
||||||
|
echo "==> FAIL: QEMU exited prematurely"
|
||||||
|
echo " Last 20 lines of serial output:"
|
||||||
|
tail -20 "$SERIAL_LOG" 2>/dev/null || echo " (no output)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
ELAPSED=$((ELAPSED + 2))
|
||||||
|
printf "\r Elapsed: %ds / %ds" "$ELAPSED" "$TIMEOUT"
|
||||||
|
done
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Kill QEMU
|
||||||
|
kill "$QEMU_PID" 2>/dev/null || true
|
||||||
|
wait "$QEMU_PID" 2>/dev/null || true
|
||||||
|
QEMU_PID=""
|
||||||
|
|
||||||
|
if [ "$SUCCESS" = "1" ]; then
|
||||||
|
echo "==> ARM64 Boot Test PASSED (${ELAPSED}s)"
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
echo "==> ARM64 Boot Test FAILED (timeout ${TIMEOUT}s)"
|
||||||
|
echo ""
|
||||||
|
echo "==> Last 30 lines of serial output:"
|
||||||
|
tail -30 "$SERIAL_LOG" 2>/dev/null || echo " (no output)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
@@ -5,17 +5,25 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
ISO="${1:?Usage: $0 <path-to-iso>}"
|
ISO="${1:?Usage: $0 <path-to-iso>}"
|
||||||
TIMEOUT_BOOT=120 # seconds to wait for boot success marker
|
TIMEOUT_BOOT=${TIMEOUT_BOOT:-120} # seconds to wait for boot success marker
|
||||||
SERIAL_LOG=$(mktemp /tmp/kubesolo-boot-test-XXXXXX.log)
|
SERIAL_LOG=$(mktemp /tmp/kubesolo-boot-test-XXXXXX.log)
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
. "$SCRIPT_DIR/../lib/qemu-helpers.sh"
|
||||||
|
|
||||||
# Temp data disk
|
# Temp data disk
|
||||||
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
DATA_DISK=$(mktemp /tmp/kubesolo-data-XXXXXX.img)
|
||||||
dd if=/dev/zero of="$DATA_DISK" bs=1M count=512 2>/dev/null
|
dd if=/dev/zero of="$DATA_DISK" bs=1M count=512 2>/dev/null
|
||||||
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
mkfs.ext4 -q -L KSOLODATA "$DATA_DISK" 2>/dev/null
|
||||||
|
|
||||||
|
QEMU_PID=""
|
||||||
|
EXTRACT_DIR=""
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
kill "$QEMU_PID" 2>/dev/null || true
|
[ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
|
||||||
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
rm -f "$DATA_DISK" "$SERIAL_LOG"
|
||||||
|
[ -n "$EXTRACT_DIR" ] && rm -rf "$EXTRACT_DIR"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
@@ -23,16 +31,25 @@ echo "==> Boot test: $ISO"
|
|||||||
echo " Timeout: ${TIMEOUT_BOOT}s"
|
echo " Timeout: ${TIMEOUT_BOOT}s"
|
||||||
echo " Serial log: $SERIAL_LOG"
|
echo " Serial log: $SERIAL_LOG"
|
||||||
|
|
||||||
# Launch QEMU in background
|
# Extract kernel from ISO
|
||||||
|
EXTRACT_DIR="$(mktemp -d /tmp/kubesolo-extract-XXXXXX)"
|
||||||
|
extract_kernel_from_iso "$ISO" "$EXTRACT_DIR"
|
||||||
|
|
||||||
|
KVM_FLAG=$(detect_kvm)
|
||||||
|
[ -n "$KVM_FLAG" ] && echo " KVM acceleration: enabled"
|
||||||
|
|
||||||
|
# Launch QEMU in background with direct kernel boot
|
||||||
|
# shellcheck disable=SC2086
|
||||||
qemu-system-x86_64 \
|
qemu-system-x86_64 \
|
||||||
-m 2048 -smp 2 \
|
-m 2048 -smp 2 \
|
||||||
-nographic \
|
-nographic \
|
||||||
-cdrom "$ISO" \
|
$KVM_FLAG \
|
||||||
-boot d \
|
-kernel "$VMLINUZ" \
|
||||||
|
-initrd "$INITRAMFS" \
|
||||||
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
-drive "file=$DATA_DISK,format=raw,if=virtio" \
|
||||||
-net nic,model=virtio \
|
-net "nic,model=virtio" \
|
||||||
-net user \
|
-net user \
|
||||||
-serial file:"$SERIAL_LOG" \
|
-serial "file:$SERIAL_LOG" \
|
||||||
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug" \
|
-append "console=ttyS0,115200n8 kubesolo.data=/dev/vda kubesolo.debug" \
|
||||||
&
|
&
|
||||||
QEMU_PID=$!
|
QEMU_PID=$!
|
||||||
@@ -41,7 +58,7 @@ QEMU_PID=$!
|
|||||||
echo " Waiting for boot..."
|
echo " Waiting for boot..."
|
||||||
ELAPSED=0
|
ELAPSED=0
|
||||||
while [ "$ELAPSED" -lt "$TIMEOUT_BOOT" ]; do
|
while [ "$ELAPSED" -lt "$TIMEOUT_BOOT" ]; do
|
||||||
if grep -q "\[kubesolo-init\] \[OK\] Stage 90-kubesolo.sh complete" "$SERIAL_LOG" 2>/dev/null; then
|
if grep -q "\[kubesolo-init\] \[OK\] KubeSolo is running" "$SERIAL_LOG" 2>/dev/null; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "==> PASS: KubeSolo OS booted successfully in ${ELAPSED}s"
|
echo "==> PASS: KubeSolo OS booted successfully in ${ELAPSED}s"
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/grubenv"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Activate switches the boot target to the passive partition.
|
// Activate switches the boot target to the passive partition.
|
||||||
@@ -12,7 +10,7 @@ import (
|
|||||||
// with boot_counter=3. If health checks fail 3 times, GRUB auto-rolls back.
|
// with boot_counter=3. If health checks fail 3 times, GRUB auto-rolls back.
|
||||||
func Activate(args []string) error {
|
func Activate(args []string) error {
|
||||||
opts := parseOpts(args)
|
opts := parseOpts(args)
|
||||||
env := grubenv.New(opts.GrubenvPath)
|
env := opts.NewBootEnv()
|
||||||
|
|
||||||
// Get passive slot (the one we want to boot into)
|
// Get passive slot (the one we want to boot into)
|
||||||
passiveSlot, err := env.PassiveSlot()
|
passiveSlot, err := env.PassiveSlot()
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/grubenv"
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/image"
|
"github.com/portainer/kubesolo-os/update/pkg/image"
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/partition"
|
"github.com/portainer/kubesolo-os/update/pkg/partition"
|
||||||
)
|
)
|
||||||
@@ -18,7 +17,7 @@ func Apply(args []string) error {
|
|||||||
return fmt.Errorf("--server is required")
|
return fmt.Errorf("--server is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
env := grubenv.New(opts.GrubenvPath)
|
env := opts.NewBootEnv()
|
||||||
|
|
||||||
// Determine passive slot
|
// Determine passive slot
|
||||||
passiveSlot, err := env.PassiveSlot()
|
passiveSlot, err := env.PassiveSlot()
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/grubenv"
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/image"
|
"github.com/portainer/kubesolo-os/update/pkg/image"
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/partition"
|
"github.com/portainer/kubesolo-os/update/pkg/partition"
|
||||||
)
|
)
|
||||||
@@ -19,7 +18,7 @@ func Check(args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get current version from active partition
|
// Get current version from active partition
|
||||||
env := grubenv.New(opts.GrubenvPath)
|
env := opts.NewBootEnv()
|
||||||
activeSlot, err := env.ActiveSlot()
|
activeSlot, err := env.ActiveSlot()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("reading active slot: %w", err)
|
return fmt.Errorf("reading active slot: %w", err)
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/grubenv"
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/health"
|
"github.com/portainer/kubesolo-os/update/pkg/health"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,7 +14,7 @@ import (
|
|||||||
// init script) to confirm the system is healthy.
|
// init script) to confirm the system is healthy.
|
||||||
func Healthcheck(args []string) error {
|
func Healthcheck(args []string) error {
|
||||||
opts := parseOpts(args)
|
opts := parseOpts(args)
|
||||||
env := grubenv.New(opts.GrubenvPath)
|
env := opts.NewBootEnv()
|
||||||
|
|
||||||
// Check if already marked successful
|
// Check if already marked successful
|
||||||
success, err := env.BootSuccess()
|
success, err := env.BootSuccess()
|
||||||
|
|||||||
@@ -1,11 +1,27 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/portainer/kubesolo-os/update/pkg/bootenv"
|
||||||
|
)
|
||||||
|
|
||||||
// opts holds shared command-line options for all subcommands.
|
// opts holds shared command-line options for all subcommands.
|
||||||
type opts struct {
|
type opts struct {
|
||||||
ServerURL string
|
ServerURL string
|
||||||
GrubenvPath string
|
GrubenvPath string
|
||||||
TimeoutSecs int
|
TimeoutSecs int
|
||||||
PubKeyPath string
|
PubKeyPath string
|
||||||
|
BootEnvType string // "grub" or "rpi"
|
||||||
|
BootEnvPath string // path for RPi boot control dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBootEnv creates a BootEnv from the parsed options.
|
||||||
|
func (o opts) NewBootEnv() bootenv.BootEnv {
|
||||||
|
switch o.BootEnvType {
|
||||||
|
case "rpi":
|
||||||
|
return bootenv.NewRPi(o.BootEnvPath)
|
||||||
|
default:
|
||||||
|
return bootenv.NewGRUB(o.GrubenvPath)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseOpts extracts command-line flags from args.
|
// parseOpts extracts command-line flags from args.
|
||||||
@@ -14,6 +30,7 @@ func parseOpts(args []string) opts {
|
|||||||
o := opts{
|
o := opts{
|
||||||
GrubenvPath: "/boot/grub/grubenv",
|
GrubenvPath: "/boot/grub/grubenv",
|
||||||
TimeoutSecs: 120,
|
TimeoutSecs: 120,
|
||||||
|
BootEnvType: "grub",
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < len(args); i++ {
|
for i := 0; i < len(args); i++ {
|
||||||
@@ -46,6 +63,16 @@ func parseOpts(args []string) opts {
|
|||||||
o.PubKeyPath = args[i+1]
|
o.PubKeyPath = args[i+1]
|
||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
case "--bootenv":
|
||||||
|
if i+1 < len(args) {
|
||||||
|
o.BootEnvType = args[i+1]
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case "--bootenv-path":
|
||||||
|
if i+1 < len(args) {
|
||||||
|
o.BootEnvPath = args[i+1]
|
||||||
|
i++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,15 +3,13 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/grubenv"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Rollback forces an immediate switch to the other partition.
|
// Rollback forces an immediate switch to the other partition.
|
||||||
// Use this to manually revert to the previous version.
|
// Use this to manually revert to the previous version.
|
||||||
func Rollback(args []string) error {
|
func Rollback(args []string) error {
|
||||||
opts := parseOpts(args)
|
opts := parseOpts(args)
|
||||||
env := grubenv.New(opts.GrubenvPath)
|
env := opts.NewBootEnv()
|
||||||
|
|
||||||
activeSlot, err := env.ActiveSlot()
|
activeSlot, err := env.ActiveSlot()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -2,42 +2,50 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/portainer/kubesolo-os/update/pkg/grubenv"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Status displays the current A/B slot configuration and boot state.
|
// Status displays the current A/B slot configuration and boot state.
|
||||||
func Status(args []string) error {
|
func Status(args []string) error {
|
||||||
opts := parseOpts(args)
|
opts := parseOpts(args)
|
||||||
env := grubenv.New(opts.GrubenvPath)
|
env := opts.NewBootEnv()
|
||||||
|
|
||||||
vars, err := env.ReadAll()
|
activeSlot, err := env.ActiveSlot()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("reading GRUB environment: %w", err)
|
return fmt.Errorf("reading active slot: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
activeSlot := vars["active_slot"]
|
passiveSlot, err := env.PassiveSlot()
|
||||||
bootCounter := vars["boot_counter"]
|
if err != nil {
|
||||||
bootSuccess := vars["boot_success"]
|
return fmt.Errorf("reading passive slot: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
passiveSlot := "B"
|
bootCounter, err := env.BootCounter()
|
||||||
if activeSlot == "B" {
|
if err != nil {
|
||||||
passiveSlot = "A"
|
return fmt.Errorf("reading boot counter: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bootSuccess, err := env.BootSuccess()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading boot success: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("KubeSolo OS — A/B Partition Status")
|
fmt.Println("KubeSolo OS — A/B Partition Status")
|
||||||
fmt.Println("───────────────────────────────────")
|
fmt.Println("───────────────────────────────────")
|
||||||
fmt.Printf(" Active slot: %s\n", activeSlot)
|
fmt.Printf(" Active slot: %s\n", activeSlot)
|
||||||
fmt.Printf(" Passive slot: %s\n", passiveSlot)
|
fmt.Printf(" Passive slot: %s\n", passiveSlot)
|
||||||
fmt.Printf(" Boot counter: %s\n", bootCounter)
|
fmt.Printf(" Boot counter: %d\n", bootCounter)
|
||||||
fmt.Printf(" Boot success: %s\n", bootSuccess)
|
if bootSuccess {
|
||||||
|
fmt.Printf(" Boot success: 1\n")
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" Boot success: 0\n")
|
||||||
|
}
|
||||||
|
|
||||||
if bootSuccess == "1" {
|
if bootSuccess {
|
||||||
fmt.Println("\n ✓ System is healthy (boot confirmed)")
|
fmt.Println("\n ✓ System is healthy (boot confirmed)")
|
||||||
} else if bootCounter == "0" {
|
} else if bootCounter == 0 {
|
||||||
fmt.Println("\n ✗ Boot counter exhausted — rollback will occur on next reboot")
|
fmt.Println("\n ✗ Boot counter exhausted — rollback will occur on next reboot")
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("\n ⚠ Boot pending verification (%s attempts remaining)\n", bootCounter)
|
fmt.Printf("\n ⚠ Boot pending verification (%d attempts remaining)\n", bootCounter)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
27
update/pkg/bootenv/bootenv.go
Normal file
27
update/pkg/bootenv/bootenv.go
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
// Package bootenv provides a platform-independent interface for managing
|
||||||
|
// A/B boot environments. It abstracts GRUB (x86_64) and RPi firmware
|
||||||
|
// (ARM64) behind a common interface.
|
||||||
|
package bootenv
|
||||||
|
|
||||||
|
// BootEnv provides read/write access to A/B boot environment variables.
|
||||||
|
type BootEnv interface {
|
||||||
|
// ActiveSlot returns the currently active boot slot ("A" or "B").
|
||||||
|
ActiveSlot() (string, error)
|
||||||
|
// PassiveSlot returns the currently passive boot slot.
|
||||||
|
PassiveSlot() (string, error)
|
||||||
|
// BootCounter returns the current boot counter value.
|
||||||
|
BootCounter() (int, error)
|
||||||
|
// BootSuccess returns whether the last boot was marked successful.
|
||||||
|
BootSuccess() (bool, error)
|
||||||
|
// MarkBootSuccess marks the current boot as successful.
|
||||||
|
MarkBootSuccess() error
|
||||||
|
// ActivateSlot switches the active boot slot and resets the counter.
|
||||||
|
ActivateSlot(slot string) error
|
||||||
|
// ForceRollback switches to the other slot immediately.
|
||||||
|
ForceRollback() error
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
SlotA = "A"
|
||||||
|
SlotB = "B"
|
||||||
|
)
|
||||||
533
update/pkg/bootenv/bootenv_test.go
Normal file
533
update/pkg/bootenv/bootenv_test.go
Normal file
@@ -0,0 +1,533 @@
|
|||||||
|
package bootenv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// createTestGrubenv writes a properly formatted 1024-byte grubenv file.
|
||||||
|
func createTestGrubenv(t *testing.T, dir string, vars map[string]string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(dir, "grubenv")
|
||||||
|
|
||||||
|
var sb strings.Builder
|
||||||
|
sb.WriteString("# GRUB Environment Block\n")
|
||||||
|
for k, v := range vars {
|
||||||
|
sb.WriteString(k + "=" + v + "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
content := sb.String()
|
||||||
|
padding := 1024 - len(content)
|
||||||
|
if padding > 0 {
|
||||||
|
content += strings.Repeat("#", padding)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// TestGRUBActiveSlot verifies ActiveSlot reads the correct value.
|
||||||
|
func TestGRUBActiveSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "1",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
slot, err := env.ActiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if slot != "A" {
|
||||||
|
t.Errorf("expected A, got %s", slot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBPassiveSlot verifies PassiveSlot returns the opposite slot.
|
||||||
|
func TestGRUBPassiveSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "0",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
passive, err := env.PassiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if passive != "B" {
|
||||||
|
t.Errorf("expected B, got %s", passive)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBBootCounter verifies BootCounter reads the correct value.
|
||||||
|
func TestGRUBBootCounter(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "2",
|
||||||
|
"boot_success": "0",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
counter, err := env.BootCounter()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if counter != 2 {
|
||||||
|
t.Errorf("expected 2, got %d", counter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBBootSuccess verifies BootSuccess reads the correct value.
|
||||||
|
func TestGRUBBootSuccess(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "1",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
success, err := env.BootSuccess()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !success {
|
||||||
|
t.Error("expected true, got false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBMarkBootSuccess verifies marking boot as successful.
|
||||||
|
func TestGRUBMarkBootSuccess(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "B",
|
||||||
|
"boot_counter": "1",
|
||||||
|
"boot_success": "0",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
if err := env.MarkBootSuccess(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
success, err := env.BootSuccess()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !success {
|
||||||
|
t.Error("expected boot_success=true after MarkBootSuccess")
|
||||||
|
}
|
||||||
|
|
||||||
|
counter, err := env.BootCounter()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if counter != 3 {
|
||||||
|
t.Errorf("expected boot_counter=3 after MarkBootSuccess, got %d", counter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBActivateSlot verifies slot activation sets correct state.
|
||||||
|
func TestGRUBActivateSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "1",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
if err := env.ActivateSlot("B"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
slot, _ := env.ActiveSlot()
|
||||||
|
if slot != "B" {
|
||||||
|
t.Errorf("expected B, got %s", slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
counter, _ := env.BootCounter()
|
||||||
|
if counter != 3 {
|
||||||
|
t.Errorf("expected counter=3, got %d", counter)
|
||||||
|
}
|
||||||
|
|
||||||
|
success, _ := env.BootSuccess()
|
||||||
|
if success {
|
||||||
|
t.Error("expected boot_success=false after ActivateSlot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBForceRollback verifies rollback switches to passive slot.
|
||||||
|
func TestGRUBForceRollback(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "1",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
if err := env.ForceRollback(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
slot, _ := env.ActiveSlot()
|
||||||
|
if slot != "B" {
|
||||||
|
t.Errorf("expected B after rollback from A, got %s", slot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBSlotCycling verifies A->B->A slot switching.
|
||||||
|
func TestGRUBSlotCycling(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "1",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
|
||||||
|
// A -> B
|
||||||
|
if err := env.ActivateSlot("B"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
slot, _ := env.ActiveSlot()
|
||||||
|
if slot != "B" {
|
||||||
|
t.Fatalf("expected B, got %s", slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
// B -> A
|
||||||
|
if err := env.ActivateSlot("A"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
slot, _ = env.ActiveSlot()
|
||||||
|
if slot != "A" {
|
||||||
|
t.Fatalf("expected A, got %s", slot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGRUBActivateInvalidSlot verifies invalid slot is rejected.
|
||||||
|
func TestGRUBActivateInvalidSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "0",
|
||||||
|
})
|
||||||
|
|
||||||
|
env := NewGRUB(path)
|
||||||
|
if err := env.ActivateSlot("C"); err == nil {
|
||||||
|
t.Fatal("expected error for invalid slot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiActiveSlot verifies ActiveSlot reads from autoboot.txt.
|
||||||
|
func TestRPiActiveSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 3, false)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
slot, err := env.ActiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if slot != "A" {
|
||||||
|
t.Errorf("expected A (partition 2), got %s", slot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiActiveSlotB verifies slot B with partition 3.
|
||||||
|
func TestRPiActiveSlotB(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 3, 2, 3, true)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
slot, err := env.ActiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if slot != "B" {
|
||||||
|
t.Errorf("expected B (partition 3), got %s", slot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiPassiveSlot verifies passive slot is opposite of active.
|
||||||
|
func TestRPiPassiveSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 3, false)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
passive, err := env.PassiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if passive != "B" {
|
||||||
|
t.Errorf("expected B, got %s", passive)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiBootCounter verifies counter is read from status file.
|
||||||
|
func TestRPiBootCounter(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 2, false)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
counter, err := env.BootCounter()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if counter != 2 {
|
||||||
|
t.Errorf("expected 2, got %d", counter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiBootCounterMissingFile verifies default when status file is absent.
|
||||||
|
func TestRPiBootCounterMissingFile(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
// Only create autoboot.txt, no boot-status
|
||||||
|
autoboot := "[all]\ntryboot_a_b=1\nboot_partition=2\n[tryboot]\nboot_partition=3\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "autoboot.txt"), []byte(autoboot), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
counter, err := env.BootCounter()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if counter != 3 {
|
||||||
|
t.Errorf("expected default counter 3, got %d", counter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiBootSuccess verifies success is read from status file.
|
||||||
|
func TestRPiBootSuccess(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 3, true)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
success, err := env.BootSuccess()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !success {
|
||||||
|
t.Error("expected true, got false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiMarkBootSuccess verifies marking boot success updates both files.
|
||||||
|
func TestRPiMarkBootSuccess(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 1, false)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
if err := env.MarkBootSuccess(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active slot should still be A
|
||||||
|
slot, _ := env.ActiveSlot()
|
||||||
|
if slot != "A" {
|
||||||
|
t.Errorf("expected active slot A, got %s", slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Boot success should be true
|
||||||
|
success, _ := env.BootSuccess()
|
||||||
|
if !success {
|
||||||
|
t.Error("expected boot_success=true after MarkBootSuccess")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Counter should be reset to 3
|
||||||
|
counter, _ := env.BootCounter()
|
||||||
|
if counter != 3 {
|
||||||
|
t.Errorf("expected counter=3 after MarkBootSuccess, got %d", counter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// [all] boot_partition should be 2 (slot A, making it permanent)
|
||||||
|
data, _ := os.ReadFile(filepath.Join(dir, "autoboot.txt"))
|
||||||
|
if !strings.Contains(string(data), "boot_partition=2") {
|
||||||
|
t.Error("expected [all] boot_partition=2 after MarkBootSuccess")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiActivateSlot verifies slot activation updates tryboot and status.
|
||||||
|
func TestRPiActivateSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 3, true)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
if err := env.ActivateSlot("B"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// [tryboot] should now point to partition 3 (slot B)
|
||||||
|
data, _ := os.ReadFile(filepath.Join(dir, "autoboot.txt"))
|
||||||
|
content := string(data)
|
||||||
|
// Find [tryboot] section and check boot_partition
|
||||||
|
idx := strings.Index(content, "[tryboot]")
|
||||||
|
if idx < 0 {
|
||||||
|
t.Fatal("missing [tryboot] section")
|
||||||
|
}
|
||||||
|
trybootSection := content[idx:]
|
||||||
|
if !strings.Contains(trybootSection, "boot_partition=3") {
|
||||||
|
t.Errorf("expected [tryboot] boot_partition=3, got: %s", trybootSection)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status should be reset
|
||||||
|
success, _ := env.BootSuccess()
|
||||||
|
if success {
|
||||||
|
t.Error("expected boot_success=false after ActivateSlot")
|
||||||
|
}
|
||||||
|
counter, _ := env.BootCounter()
|
||||||
|
if counter != 3 {
|
||||||
|
t.Errorf("expected counter=3, got %d", counter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiActivateInvalidSlot verifies invalid slot is rejected.
|
||||||
|
func TestRPiActivateInvalidSlot(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 3, false)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
if err := env.ActivateSlot("C"); err == nil {
|
||||||
|
t.Fatal("expected error for invalid slot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiForceRollback verifies rollback swaps the active slot.
|
||||||
|
func TestRPiForceRollback(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 3, true)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
if err := env.ForceRollback(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// [all] should now point to partition 3 (slot B)
|
||||||
|
slot, _ := env.ActiveSlot()
|
||||||
|
if slot != "B" {
|
||||||
|
t.Errorf("expected B after rollback from A, got %s", slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Success should be false
|
||||||
|
success, _ := env.BootSuccess()
|
||||||
|
if success {
|
||||||
|
t.Error("expected boot_success=false after ForceRollback")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRPiSlotCycling verifies A->B->A slot switching works.
|
||||||
|
func TestRPiSlotCycling(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, dir, 2, 3, 3, true)
|
||||||
|
|
||||||
|
env := NewRPi(dir)
|
||||||
|
|
||||||
|
// Rollback A -> B
|
||||||
|
if err := env.ForceRollback(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
slot, _ := env.ActiveSlot()
|
||||||
|
if slot != "B" {
|
||||||
|
t.Fatalf("expected B, got %s", slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rollback B -> A
|
||||||
|
if err := env.ForceRollback(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
slot, _ = env.ActiveSlot()
|
||||||
|
if slot != "A" {
|
||||||
|
t.Fatalf("expected A, got %s", slot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestInterfaceCompliance verifies both implementations satisfy BootEnv.
|
||||||
|
func TestInterfaceCompliance(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
grubPath := createTestGrubenv(t, dir, map[string]string{
|
||||||
|
"active_slot": "A",
|
||||||
|
"boot_counter": "3",
|
||||||
|
"boot_success": "0",
|
||||||
|
})
|
||||||
|
|
||||||
|
rpiDir := t.TempDir()
|
||||||
|
createTestAutobootFiles(t, rpiDir, 2, 3, 3, false)
|
||||||
|
|
||||||
|
impls := map[string]BootEnv{
|
||||||
|
"grub": NewGRUB(grubPath),
|
||||||
|
"rpi": NewRPi(rpiDir),
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, env := range impls {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
slot, err := env.ActiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ActiveSlot: %v", err)
|
||||||
|
}
|
||||||
|
if slot != "A" {
|
||||||
|
t.Errorf("ActiveSlot: expected A, got %s", slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
passive, err := env.PassiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PassiveSlot: %v", err)
|
||||||
|
}
|
||||||
|
if passive != "B" {
|
||||||
|
t.Errorf("PassiveSlot: expected B, got %s", passive)
|
||||||
|
}
|
||||||
|
|
||||||
|
counter, err := env.BootCounter()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BootCounter: %v", err)
|
||||||
|
}
|
||||||
|
if counter != 3 {
|
||||||
|
t.Errorf("BootCounter: expected 3, got %d", counter)
|
||||||
|
}
|
||||||
|
|
||||||
|
success, err := env.BootSuccess()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BootSuccess: %v", err)
|
||||||
|
}
|
||||||
|
if success {
|
||||||
|
t.Error("BootSuccess: expected false")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// createTestAutobootFiles is a helper that writes both autoboot.txt and boot-status.
|
||||||
|
func createTestAutobootFiles(t *testing.T, dir string, allPart, trybootPart, counter int, success bool) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
autoboot := "[all]\ntryboot_a_b=1\nboot_partition=" + strconv.Itoa(allPart) + "\n"
|
||||||
|
autoboot += "[tryboot]\nboot_partition=" + strconv.Itoa(trybootPart) + "\n"
|
||||||
|
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "autoboot.txt"), []byte(autoboot), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
successVal := "0"
|
||||||
|
if success {
|
||||||
|
successVal = "1"
|
||||||
|
}
|
||||||
|
status := "boot_counter=" + strconv.Itoa(counter) + "\nboot_success=" + successVal + "\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "boot-status"), []byte(status), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
23
update/pkg/bootenv/grub.go
Normal file
23
update/pkg/bootenv/grub.go
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
package bootenv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/portainer/kubesolo-os/update/pkg/grubenv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GRUBEnv implements BootEnv using GRUB environment variables.
|
||||||
|
type GRUBEnv struct {
|
||||||
|
env *grubenv.Env
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGRUB creates a new GRUB-based BootEnv.
|
||||||
|
func NewGRUB(path string) BootEnv {
|
||||||
|
return &GRUBEnv{env: grubenv.New(path)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GRUBEnv) ActiveSlot() (string, error) { return g.env.ActiveSlot() }
|
||||||
|
func (g *GRUBEnv) PassiveSlot() (string, error) { return g.env.PassiveSlot() }
|
||||||
|
func (g *GRUBEnv) BootCounter() (int, error) { return g.env.BootCounter() }
|
||||||
|
func (g *GRUBEnv) BootSuccess() (bool, error) { return g.env.BootSuccess() }
|
||||||
|
func (g *GRUBEnv) MarkBootSuccess() error { return g.env.MarkBootSuccess() }
|
||||||
|
func (g *GRUBEnv) ActivateSlot(slot string) error { return g.env.ActivateSlot(slot) }
|
||||||
|
func (g *GRUBEnv) ForceRollback() error { return g.env.ForceRollback() }
|
||||||
267
update/pkg/bootenv/rpi.go
Normal file
267
update/pkg/bootenv/rpi.go
Normal file
@@ -0,0 +1,267 @@
|
|||||||
|
package bootenv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// RPi partition numbers: slot A = partition 2, slot B = partition 3.
|
||||||
|
rpiSlotAPartition = 2
|
||||||
|
rpiSlotBPartition = 3
|
||||||
|
|
||||||
|
defaultBootCounter = 3
|
||||||
|
)
|
||||||
|
|
||||||
|
// RPiEnv implements BootEnv using Raspberry Pi firmware autoboot.txt.
|
||||||
|
type RPiEnv struct {
|
||||||
|
autobootPath string // path to autoboot.txt
|
||||||
|
statusPath string // path to boot-status file
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRPi creates a new RPi-based BootEnv.
|
||||||
|
// dir is the directory containing autoboot.txt (typically the boot control
|
||||||
|
// partition mount point).
|
||||||
|
func NewRPi(dir string) BootEnv {
|
||||||
|
return &RPiEnv{
|
||||||
|
autobootPath: filepath.Join(dir, "autoboot.txt"),
|
||||||
|
statusPath: filepath.Join(dir, "boot-status"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RPiEnv) ActiveSlot() (string, error) {
|
||||||
|
partNum, err := r.readAllBootPartition()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("reading active slot: %w", err)
|
||||||
|
}
|
||||||
|
return partNumToSlot(partNum)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RPiEnv) PassiveSlot() (string, error) {
|
||||||
|
active, err := r.ActiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if active == SlotA {
|
||||||
|
return SlotB, nil
|
||||||
|
}
|
||||||
|
return SlotA, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RPiEnv) BootCounter() (int, error) {
|
||||||
|
status, err := r.readStatus()
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
val, ok := status["boot_counter"]
|
||||||
|
if !ok {
|
||||||
|
return defaultBootCounter, nil
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(val)
|
||||||
|
if err != nil {
|
||||||
|
return -1, fmt.Errorf("invalid boot_counter %q: %w", val, err)
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RPiEnv) BootSuccess() (bool, error) {
|
||||||
|
status, err := r.readStatus()
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return status["boot_success"] == "1", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RPiEnv) MarkBootSuccess() error {
|
||||||
|
// Make the current slot permanent by updating [all] boot_partition
|
||||||
|
active, err := r.ActiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marking boot success: %w", err)
|
||||||
|
}
|
||||||
|
partNum := slotToPartNum(active)
|
||||||
|
if err := r.writeAllBootPartition(partNum); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return r.writeStatus(defaultBootCounter, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RPiEnv) ActivateSlot(slot string) error {
|
||||||
|
if slot != SlotA && slot != SlotB {
|
||||||
|
return fmt.Errorf("invalid slot: %q (must be A or B)", slot)
|
||||||
|
}
|
||||||
|
partNum := slotToPartNum(slot)
|
||||||
|
// Update [tryboot] to point to the new slot
|
||||||
|
if err := r.writeTrybootPartition(partNum); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return r.writeStatus(defaultBootCounter, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RPiEnv) ForceRollback() error {
|
||||||
|
passive, err := r.PassiveSlot()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Swap the [all] boot_partition to the other slot
|
||||||
|
partNum := slotToPartNum(passive)
|
||||||
|
if err := r.writeAllBootPartition(partNum); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := r.writeTrybootPartition(partNum); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return r.writeStatus(defaultBootCounter, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readAllBootPartition reads the boot_partition value from the [all] section.
|
||||||
|
func (r *RPiEnv) readAllBootPartition() (int, error) {
|
||||||
|
sections, err := r.parseAutoboot()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
val, ok := sections["all"]["boot_partition"]
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("boot_partition not found in [all] section")
|
||||||
|
}
|
||||||
|
return strconv.Atoi(val)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeAllBootPartition updates the [all] boot_partition value.
|
||||||
|
func (r *RPiEnv) writeAllBootPartition(partNum int) error {
|
||||||
|
sections, err := r.parseAutoboot()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sections["all"] == nil {
|
||||||
|
sections["all"] = make(map[string]string)
|
||||||
|
}
|
||||||
|
sections["all"]["boot_partition"] = strconv.Itoa(partNum)
|
||||||
|
return r.writeAutoboot(sections)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeTrybootPartition updates the [tryboot] boot_partition value.
|
||||||
|
func (r *RPiEnv) writeTrybootPartition(partNum int) error {
|
||||||
|
sections, err := r.parseAutoboot()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sections["tryboot"] == nil {
|
||||||
|
sections["tryboot"] = make(map[string]string)
|
||||||
|
}
|
||||||
|
sections["tryboot"]["boot_partition"] = strconv.Itoa(partNum)
|
||||||
|
return r.writeAutoboot(sections)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseAutoboot reads autoboot.txt into a map of section -> key=value pairs.
|
||||||
|
func (r *RPiEnv) parseAutoboot() (map[string]map[string]string, error) {
|
||||||
|
data, err := os.ReadFile(r.autobootPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading autoboot.txt: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sections := make(map[string]map[string]string)
|
||||||
|
currentSection := ""
|
||||||
|
|
||||||
|
for _, line := range strings.Split(string(data), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
|
||||||
|
currentSection = line[1 : len(line)-1]
|
||||||
|
if sections[currentSection] == nil {
|
||||||
|
sections[currentSection] = make(map[string]string)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parts := strings.SplitN(line, "=", 2)
|
||||||
|
if len(parts) == 2 && currentSection != "" {
|
||||||
|
sections[currentSection][strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sections, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeAutoboot writes sections back to autoboot.txt.
|
||||||
|
// Section order: [all] first, then [tryboot].
|
||||||
|
func (r *RPiEnv) writeAutoboot(sections map[string]map[string]string) error {
|
||||||
|
var sb strings.Builder
|
||||||
|
|
||||||
|
// Write [all] section first
|
||||||
|
if all, ok := sections["all"]; ok {
|
||||||
|
sb.WriteString("[all]\n")
|
||||||
|
for k, v := range all {
|
||||||
|
sb.WriteString(k + "=" + v + "\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write [tryboot] section
|
||||||
|
if tryboot, ok := sections["tryboot"]; ok {
|
||||||
|
sb.WriteString("[tryboot]\n")
|
||||||
|
for k, v := range tryboot {
|
||||||
|
sb.WriteString(k + "=" + v + "\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return os.WriteFile(r.autobootPath, []byte(sb.String()), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readStatus reads the boot-status key=value file.
|
||||||
|
func (r *RPiEnv) readStatus() (map[string]string, error) {
|
||||||
|
data, err := os.ReadFile(r.statusPath)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
// Return defaults if status file doesn't exist yet
|
||||||
|
return map[string]string{
|
||||||
|
"boot_counter": strconv.Itoa(defaultBootCounter),
|
||||||
|
"boot_success": "0",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("reading boot-status: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
status := make(map[string]string)
|
||||||
|
for _, line := range strings.Split(string(data), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parts := strings.SplitN(line, "=", 2)
|
||||||
|
if len(parts) == 2 {
|
||||||
|
status[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeStatus writes boot_counter and boot_success to the status file.
|
||||||
|
func (r *RPiEnv) writeStatus(counter int, success bool) error {
|
||||||
|
successVal := "0"
|
||||||
|
if success {
|
||||||
|
successVal = "1"
|
||||||
|
}
|
||||||
|
content := fmt.Sprintf("boot_counter=%d\nboot_success=%s\n", counter, successVal)
|
||||||
|
return os.WriteFile(r.statusPath, []byte(content), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func partNumToSlot(partNum int) (string, error) {
|
||||||
|
switch partNum {
|
||||||
|
case rpiSlotAPartition:
|
||||||
|
return SlotA, nil
|
||||||
|
case rpiSlotBPartition:
|
||||||
|
return SlotB, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unknown partition number %d (expected %d or %d)", partNum, rpiSlotAPartition, rpiSlotBPartition)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func slotToPartNum(slot string) int {
|
||||||
|
if slot == SlotB {
|
||||||
|
return rpiSlotBPartition
|
||||||
|
}
|
||||||
|
return rpiSlotAPartition
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user