feat(update): pre-flight gates + deeper healthcheck + auto-rollback

Phase 8 of v0.3. Tightens the update lifecycle on both ends.

Pre-flight (apply.go, before any download):
- Free-space check on the passive partition: image size + 10% headroom must
  be available. Uses statfs(2) via the new pkg/partition.FreeBytes /
  HasFreeSpaceFor helpers (tests cover happy path, tiny request, huge
  request, missing path). Catches corrupted-FS and shrunk-partition cases
  before we destroy the existing slot data.
- Node-block-label check: refuses if the local K8s node carries the
  updates.kubesolo.io/block=true label. New pkg/health.CheckNodeBlocked
  shells out to kubectl per the project's zero-deps stance. Silently bypassed
  when no kubeconfig is reachable (air-gap case). Skipped by --force.

Healthcheck (extended via new pkg/health/extended.go + preflight.go):
- CheckKubeSystemReady waits until every kube-system pod has held the Running
  phase for >= N seconds (default 30). Catches "started ok, will crash-loop"
  bugs that a single-shot phase check misses.
- CheckProbeURL fetches an operator-supplied URL; 200 = pass. Wired through
  update.conf as healthcheck_url= and cloud-init updates.healthcheck_url.
- CheckDiskWritable writes/fsyncs/reads a 1-KiB probe under /var/lib/kubesolo.
  Always runs in healthcheck so a wedged data partition fails fast.
- pkg/health.Status grows KubeSystemReady, ProbeURL, DiskWritable booleans.
  Optional checks default to true in RunAll() so they don't block when
  unconfigured. health_test.go updated to the new 6-field shape.

Auto-rollback (healthcheck.go):
- state.UpdateState gains HealthCheckFailures (consecutive post-Activated
  failures). Reset on a clean pass.
- --auto-rollback-after N (also auto_rollback_after= in update.conf) triggers
  env.ForceRollback() when the failure count reaches the threshold. State
  transitions to RolledBack with a descriptive LastError. The command still
  exits with the healthcheck error; the operator/init is expected to reboot.
- Only fires while Phase == Activated. Doesn't second-guess a long-stable
  system that happens to fail one healthcheck.

config / opts / cloud-init plumbing:
- update.conf gains healthcheck_url= and auto_rollback_after= keys.
- New CLI flags: --healthcheck-url, --auto-rollback-after, --kube-system-settle.
- cloud-init full-config.yaml documents the new updates: subfields.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

update/cmd/apply.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"os"
 	"runtime"
 	"time"
 
 	"github.com/portainer/kubesolo-os/update/pkg/config"
+	"github.com/portainer/kubesolo-os/update/pkg/health"
 	"github.com/portainer/kubesolo-os/update/pkg/image"
 	"github.com/portainer/kubesolo-os/update/pkg/oci"
 	"github.com/portainer/kubesolo-os/update/pkg/partition"
@@ -71,6 +73,19 @@ func Apply(args []string) error {
 			window.String())
 	}
 
+	// Node-block-label gate — workload authors can defer an update by
+	// labeling the node updates.kubesolo.io/block=true. Skipped with --force
+	// and silently bypassed when the K8s API isn't reachable (air-gap).
+	if !opts.Force {
+		blocked, berr := health.CheckNodeBlocked("")
+		if berr != nil {
+			slog.Warn("node-block check failed, allowing update", "error", berr)
+		} else if blocked {
+			return fmt.Errorf("node carries label %s=true; refusing update (pass --force to override)",
+				health.NodeBlockLabel)
+		}
+	}
+
 	st, err := state.Load(opts.StatePath)
 	if err != nil {
 		// Don't block the operation on a corrupt state file. Log + recover.
@@ -186,6 +201,30 @@ func Apply(args []string) error {
 	}
 	defer partition.Unmount(mountPoint)
 
+	// Free-space pre-write check: the passive partition must have at least
+	// (kernel + initramfs) + 10% headroom. Catches corrupted-FS reports and
+	// shrunk/wrong-size partitions before we destroy the existing slot data.
+	var imgSize int64
+	for _, p := range []string{staged.VmlinuzPath, staged.InitramfsPath} {
+		fi, ferr := os.Stat(p)
+		if ferr != nil {
+			_ = st.RecordError(opts.StatePath, fmt.Errorf("stat staged file %s: %w", p, ferr))
+			return fmt.Errorf("stat staged file %s: %w", p, ferr)
+		}
+		imgSize += fi.Size()
+	}
+	avail, ok, ferr := partition.HasFreeSpaceFor(mountPoint, imgSize, 10)
+	if ferr != nil {
+		_ = st.RecordError(opts.StatePath, fmt.Errorf("free-space check: %w", ferr))
+		return fmt.Errorf("free-space check: %w", ferr)
+	}
+	if !ok {
+		err := fmt.Errorf("insufficient space on %s: have %.1f MiB, need %.1f MiB (image + 10%% headroom)",
+			passiveSlot, float64(avail)/(1<<20), float64(imgSize)*1.1/(1<<20))
+		_ = st.RecordError(opts.StatePath, err)
+		return err
+	}
+
 	// Write image to passive partition
 	if err := partition.WriteSystemImage(mountPoint, staged.VmlinuzPath, staged.InitramfsPath, staged.Version); err != nil {
 		_ = st.RecordError(opts.StatePath, fmt.Errorf("writing system image: %w", err))

update/cmd/healthcheck.go

@@ -17,6 +17,11 @@ import (
 // State transition: Activated → Verifying → Success on pass, → Failed on fail.
 // If state isn't in Activated (e.g. manual run on a long-stable system), the
 // state file is left alone — healthcheck still does its job.
+//
+// When --auto-rollback-after N is set, consecutive post-Activated failures
+// are counted in state.HealthCheckFailures. On the Nth failure, the agent
+// calls Rollback() and the operator is expected to reboot (this command
+// does not reboot the host — that's policy left to systemd/init).
 func Healthcheck(args []string) error {
 	opts := parseOpts(args)
 	env := opts.NewBootEnv()
@@ -48,18 +53,44 @@ func Healthcheck(args []string) error {
 
 	timeout := time.Duration(opts.TimeoutSecs) * time.Second
 	checker := health.NewChecker("", "", timeout)
+	checker.ProbeURL = opts.HealthcheckURL
+	if opts.KubeSystemSettle > 0 {
+		checker.KubeSystemSettle = time.Duration(opts.KubeSystemSettle) * time.Second
+	}
+	// Probe the data partition every healthcheck so a wedged disk fails fast.
+	checker.DataDir = "/var/lib/kubesolo"
 
-	slog.Info("running post-boot health checks", "timeout", timeout)
+	slog.Info("running post-boot health checks",
+		"timeout", timeout,
+		"probe_url", checker.ProbeURL,
+		"kube_system_settle", checker.KubeSystemSettle)
 
 	status, err := checker.WaitForHealthy()
 	if err != nil {
 		fmt.Printf("Health check FAILED: %s\n", status.Message)
-		fmt.Printf("  containerd: %v\n", status.Containerd)
-		fmt.Printf("  apiserver:  %v\n", status.APIServer)
-		fmt.Printf("  node_ready: %v\n", status.NodeReady)
+		printStatusBreakdown(status)
 		fmt.Println("\nBoot NOT marked successful — system may roll back on next reboot")
+
 		if postActivation {
+			st.HealthCheckFailures++
 			_ = st.RecordError(opts.StatePath, fmt.Errorf("post-boot health check failed: %s", status.Message))
+
+			// Auto-rollback escalation. Only trigger when post-Activated;
+			// don't second-guess a healthy long-running system.
+			if opts.AutoRollbackAfter > 0 && st.HealthCheckFailures >= opts.AutoRollbackAfter {
+				slog.Warn("auto-rollback threshold reached",
+					"failures", st.HealthCheckFailures,
+					"threshold", opts.AutoRollbackAfter)
+				if rerr := env.ForceRollback(); rerr != nil {
+					slog.Error("auto-rollback failed", "error", rerr)
+					return err // return the original healthcheck error
+				}
+				if terr := st.Transition(opts.StatePath, state.PhaseRolledBack, "",
+					fmt.Sprintf("auto-rollback after %d healthcheck failures", st.HealthCheckFailures)); terr != nil {
+					slog.Warn("state transition failed", "phase", state.PhaseRolledBack, "error", terr)
+				}
+				fmt.Println("\nAuto-rollback triggered. Reboot to complete the rollback.")
+			}
 		}
 		return err
 	}
@@ -73,15 +104,32 @@ func Healthcheck(args []string) error {
 	}
 
 	if postActivation {
+		// Reset failure counter on a clean pass.
+		st.HealthCheckFailures = 0
 		if err := st.Transition(opts.StatePath, state.PhaseSuccess, "", ""); err != nil {
 			slog.Warn("state transition failed", "phase", state.PhaseSuccess, "error", err)
 		}
 	}
 
 	fmt.Println("Health check PASSED — boot marked successful")
-	fmt.Printf("  containerd: %v\n", status.Containerd)
-	fmt.Printf("  apiserver:  %v\n", status.APIServer)
-	fmt.Printf("  node_ready: %v\n", status.NodeReady)
+	printStatusBreakdown(status)
 
 	return nil
 }
+
+// printStatusBreakdown emits a human-readable per-check summary. Only emits
+// optional check lines when they actually ran.
+func printStatusBreakdown(s *health.Status) {
+	fmt.Printf("  containerd:        %v\n", s.Containerd)
+	fmt.Printf("  apiserver:         %v\n", s.APIServer)
+	fmt.Printf("  node_ready:        %v\n", s.NodeReady)
+	if !s.KubeSystemReady {
+		fmt.Printf("  kube-system pods:  %v\n", s.KubeSystemReady)
+	}
+	if !s.ProbeURL {
+		fmt.Printf("  probe URL:         %v\n", s.ProbeURL)
+	}
+	if !s.DiskWritable {
+		fmt.Printf("  disk writable:     %v\n", s.DiskWritable)
+	}
+}

update/cmd/opts.go

@@ -18,12 +18,15 @@ type opts struct {
 	PubKeyPath        string
 	BootEnvType       string // "grub" or "rpi"
 	BootEnvPath       string // path for RPi boot control dir
-	StatePath         string // location of state.json (default: state.DefaultPath)
-	ConfPath          string // location of update.conf (default: config.DefaultPath)
-	Channel           string // update channel ("stable" by default)
-	MaintenanceWindow string // "HH:MM-HH:MM" or empty for always-allow
-	Force             bool   // bypass maintenance window
-	JSON              bool   // status: emit JSON instead of human-readable
+	StatePath          string // location of state.json (default: state.DefaultPath)
+	ConfPath           string // location of update.conf (default: config.DefaultPath)
+	Channel            string // update channel ("stable" by default)
+	MaintenanceWindow  string // "HH:MM-HH:MM" or empty for always-allow
+	HealthcheckURL     string // optional GET probe for healthcheck
+	AutoRollbackAfter  int    // healthcheck: rollback after N consecutive failures (0=off)
+	KubeSystemSettle   int    // healthcheck: kube-system pods must be Running for N seconds (0=disabled)
+	Force              bool   // bypass maintenance window
+	JSON               bool   // status: emit JSON instead of human-readable
 }
 
 // NewBootEnv creates a BootEnv from the parsed options.
@@ -74,6 +77,12 @@ func parseOpts(args []string) opts {
 		if cfg.PubKey != "" {
 			o.PubKeyPath = cfg.PubKey
 		}
+		if cfg.HealthcheckURL != "" {
+			o.HealthcheckURL = cfg.HealthcheckURL
+		}
+		if cfg.AutoRollbackAfter > 0 {
+			o.AutoRollbackAfter = cfg.AutoRollbackAfter
+		}
 	} else if err != nil {
 		slog.Warn("could not load update.conf", "path", o.ConfPath, "error", err)
 	}
@@ -100,6 +109,43 @@ func parseOpts(args []string) opts {
 			}
 		case "--force":
 			o.Force = true
+		case "--healthcheck-url":
+			if i+1 < len(args) {
+				o.HealthcheckURL = args[i+1]
+				i++
+			}
+		case "--auto-rollback-after":
+			if i+1 < len(args) {
+				n := 0
+				for _, ch := range args[i+1] {
+					if ch >= '0' && ch <= '9' {
+						n = n*10 + int(ch-'0')
+					} else {
+						n = 0
+						break
+					}
+				}
+				if n > 0 {
+					o.AutoRollbackAfter = n
+				}
+				i++
+			}
+		case "--kube-system-settle":
+			if i+1 < len(args) {
+				n := 0
+				for _, ch := range args[i+1] {
+					if ch >= '0' && ch <= '9' {
+						n = n*10 + int(ch-'0')
+					} else {
+						n = 0
+						break
+					}
+				}
+				if n > 0 {
+					o.KubeSystemSettle = n
+				}
+				i++
+			}
 		case "--json":
 			o.JSON = true
 		case "--server":