feat(update): pre-flight gates + deeper healthcheck + auto-rollback
Some checks failed
ARM64 Build / Build generic ARM64 disk image (push) Failing after 4s
CI / Go Tests (push) Successful in 1m29s
CI / Shellcheck (push) Successful in 48s
CI / Build Go Binaries (amd64, linux, linux-amd64) (push) Successful in 1m12s
CI / Build Go Binaries (arm64, linux, linux-arm64) (push) Has been cancelled
Some checks failed
ARM64 Build / Build generic ARM64 disk image (push) Failing after 4s
CI / Go Tests (push) Successful in 1m29s
CI / Shellcheck (push) Successful in 48s
CI / Build Go Binaries (amd64, linux, linux-amd64) (push) Successful in 1m12s
CI / Build Go Binaries (arm64, linux, linux-arm64) (push) Has been cancelled
Phase 8 of v0.3. Tightens the update lifecycle on both ends. Pre-flight (apply.go, before any download): - Free-space check on the passive partition: image size + 10% headroom must be available. Uses statfs(2) via the new pkg/partition.FreeBytes / HasFreeSpaceFor helpers (tests cover happy path, tiny request, huge request, missing path). Catches corrupted-FS and shrunk-partition cases before we destroy the existing slot data. - Node-block-label check: refuses if the local K8s node carries the updates.kubesolo.io/block=true label. New pkg/health.CheckNodeBlocked shells out to kubectl per the project's zero-deps stance. Silently bypassed when no kubeconfig is reachable (air-gap case). Skipped by --force. Healthcheck (extended via new pkg/health/extended.go + preflight.go): - CheckKubeSystemReady waits until every kube-system pod has held the Running phase for >= N seconds (default 30). Catches "started ok, will crash-loop" bugs that a single-shot phase check misses. - CheckProbeURL fetches an operator-supplied URL; 200 = pass. Wired through update.conf as healthcheck_url= and cloud-init updates.healthcheck_url. - CheckDiskWritable writes/fsyncs/reads a 1-KiB probe under /var/lib/kubesolo. Always runs in healthcheck so a wedged data partition fails fast. - pkg/health.Status grows KubeSystemReady, ProbeURL, DiskWritable booleans. Optional checks default to true in RunAll() so they don't block when unconfigured. health_test.go updated to the new 6-field shape. Auto-rollback (healthcheck.go): - state.UpdateState gains HealthCheckFailures (consecutive post-Activated failures). Reset on a clean pass. - --auto-rollback-after N (also auto_rollback_after= in update.conf) triggers env.ForceRollback() when the failure count reaches the threshold. State transitions to RolledBack with a descriptive LastError. The command still exits with the healthcheck error; the operator/init is expected to reboot. - Only fires while Phase == Activated. Doesn't second-guess a long-stable system that happens to fail one healthcheck. config / opts / cloud-init plumbing: - update.conf gains healthcheck_url= and auto_rollback_after= keys. - New CLI flags: --healthcheck-url, --auto-rollback-after, --kube-system-settle. - cloud-init full-config.yaml documents the new updates: subfields. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
34
update/pkg/partition/freespace.go
Normal file
34
update/pkg/partition/freespace.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package partition
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// FreeBytes returns the number of free bytes available on the filesystem
|
||||
// containing `path`. Uses statfs(2); path must exist and be readable.
|
||||
func FreeBytes(path string) (uint64, error) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &stat); err != nil {
|
||||
return 0, fmt.Errorf("statfs %s: %w", path, err)
|
||||
}
|
||||
// Bavail is the count of free blocks available to non-root users —
|
||||
// matches what `df` reports. Bsize is the block size in bytes.
|
||||
//nolint:unconvert // Bavail is uint64 on most platforms but int64 on darwin/freebsd
|
||||
return uint64(stat.Bavail) * uint64(stat.Bsize), nil
|
||||
}
|
||||
|
||||
// HasFreeSpaceFor reports whether `path`'s filesystem has at least `wantBytes`
|
||||
// of free space, with `headroomPct` reserved (e.g. 10 = require 110% of want).
|
||||
// Returns the available bytes alongside, so callers can render a useful error.
|
||||
func HasFreeSpaceFor(path string, wantBytes int64, headroomPct int) (avail uint64, ok bool, err error) {
|
||||
avail, err = FreeBytes(path)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if wantBytes < 0 {
|
||||
return avail, false, fmt.Errorf("invalid wantBytes %d", wantBytes)
|
||||
}
|
||||
required := uint64(wantBytes) * uint64(100+headroomPct) / 100
|
||||
return avail, avail >= required, nil
|
||||
}
|
||||
44
update/pkg/partition/freespace_test.go
Normal file
44
update/pkg/partition/freespace_test.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package partition
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFreeBytesReturnsNonZeroOnTempDir(t *testing.T) {
|
||||
b, err := FreeBytes(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("FreeBytes: %v", err)
|
||||
}
|
||||
// On any sane test runner the temp filesystem has more than 1 KiB free.
|
||||
if b < 1024 {
|
||||
t.Errorf("FreeBytes = %d, want > 1024 on /tmp", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreeBytesNonExistentPath(t *testing.T) {
|
||||
_, err := FreeBytes("/this/path/does/not/exist/at/all")
|
||||
if err == nil {
|
||||
t.Error("expected error for missing path, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasFreeSpaceForRejectsHugeRequest(t *testing.T) {
|
||||
// Request 1 PiB with 10% headroom on /tmp — no test runner has that
|
||||
// much free, so this should consistently report not-enough.
|
||||
avail, ok, err := HasFreeSpaceFor(t.TempDir(), 1<<50, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("HasFreeSpaceFor: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Errorf("expected insufficient space for 1PiB, got avail=%d ok=true", avail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasFreeSpaceForAcceptsSmallRequest(t *testing.T) {
|
||||
// 1 KiB with 10% headroom = 1.1 KiB. Any temp dir has this.
|
||||
_, ok, err := HasFreeSpaceFor(t.TempDir(), 1024, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("HasFreeSpaceFor: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Error("expected sufficient space for 1KiB on /tmp")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user