feat(update): persistent state machine + lifecycle metrics

Phase 5 of v0.3. Adds an explicit, on-disk state machine to the update agent
so the lifecycle of an attempt is observable end-to-end, instead of being
inferred from logs and side effects.

New package update/pkg/state:
- Phase enum (idle, checking, downloading, staged, activated, verifying,
  success, rolled_back, failed)
- UpdateState struct persisted to /var/lib/kubesolo/update/state.json
  (overridable via --state). Atomic write (.tmp + rename). Survives reboots
  and slot switches because the file lives on the data partition.
- Transition helper that bumps AttemptCount when an attempt starts, resets
  it when the target version changes, sets/clears LastError on
  failed/success transitions, and stamps StartedAt + UpdatedAt.
- 13 unit tests cover the lifecycle, atomic write, version-change reset,
  error recording, idempotent SetFromVersion, garbage-file handling.

Wired into the existing commands:
- apply.go transitions Idle -> Checking -> Downloading -> Staged, with
  RecordError on any step failure. Reads the active slot's version file to
  populate FromVersion.
- activate.go transitions to Activated.
- healthcheck.go transitions Activated -> Verifying -> Success on pass,
  or to Failed on fail. Skips transitions if state isn't post-activation
  (manual healthcheck on a stable system shouldn't churn the state).
- rollback.go transitions to RolledBack with LastError="manual rollback".
- check.go intentionally untouched — checks are passive queries, not
  attempts; they shouldn't reset AttemptCount.

status.go gains a --json mode that emits the full state report (A/B slots,
boot counter, full UpdateState) for orchestration tooling. Human-readable
mode also prints an Update Lifecycle section when state.phase != idle.

pkg/metrics gains three new series, derived from state.json at scrape time:
- kubesolo_update_phase{phase="..."} — 1 for current, 0 for all others;
  all nine phase values always emitted so dashboards see complete series
- kubesolo_update_attempts_total
- kubesolo_update_last_attempt_timestamp_seconds
Server.SetStatePath() configures the file location; defaults to absent
which emits Idle defaults. Three new tests cover the absent / active /
all-phases-emitted cases.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

update/pkg/metrics/metrics.go

@@ -11,6 +11,9 @@
 //	kubesolo_os_update_last_check_timestamp_seconds  unix timestamp (gauge)
 //	kubesolo_os_memory_total_bytes                   total RAM (gauge)
 //	kubesolo_os_memory_available_bytes               available RAM (gauge)
+//	kubesolo_update_phase{phase}                     1 for current phase, 0 for others
+//	kubesolo_update_attempts_total                   counter — attempts at current ToVersion
+//	kubesolo_update_last_attempt_timestamp_seconds   unix timestamp of last state update
 //
 // This is a zero-dependency implementation — no Prometheus client library needed.
 // It serves metrics in the Prometheus text exposition format.
@@ -25,11 +28,14 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/portainer/kubesolo-os/update/pkg/state"
 )
 
 // Server is a lightweight Prometheus metrics HTTP server.
 type Server struct {
 	grubenvPath string
+	statePath   string
 	listenAddr  string
 	startTime   time.Time
 
@@ -47,6 +53,27 @@ func NewServer(listenAddr, grubenvPath string) *Server {
 	}
 }
 
+// SetStatePath sets the location of the update state.json file. If empty or
+// unset, state-derived metrics are emitted with the Idle defaults.
+func (s *Server) SetStatePath(p string) {
+	s.statePath = p
+}
+
+// allPhases lists every Phase value we emit as a kubesolo_update_phase
+// time-series, so consumers see all label values (with value 0 for non-current
+// phases). Mirror of validPhases in pkg/state.
+var allPhases = []state.Phase{
+	state.PhaseIdle,
+	state.PhaseChecking,
+	state.PhaseDownloading,
+	state.PhaseStaged,
+	state.PhaseActivated,
+	state.PhaseVerifying,
+	state.PhaseSuccess,
+	state.PhaseRolledBack,
+	state.PhaseFailed,
+}
+
 // SetUpdateAvailable records whether an update is available.
 func (s *Server) SetUpdateAvailable(available bool) {
 	s.mu.Lock()
@@ -125,9 +152,49 @@ func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
 	sb.WriteString("# TYPE kubesolo_os_memory_available_bytes gauge\n")
 	sb.WriteString(fmt.Sprintf("kubesolo_os_memory_available_bytes %d\n", memAvail))
 
+	// Update lifecycle (from state.json)
+	s.writeUpdateStateMetrics(&sb)
+
 	fmt.Fprint(w, sb.String())
 }
 
+// writeUpdateStateMetrics appends update-lifecycle metrics derived from the
+// state.json file. If the file is missing or unreadable, emits the Idle
+// defaults so the metric series exists at all times.
+func (s *Server) writeUpdateStateMetrics(sb *strings.Builder) {
+	current := state.PhaseIdle
+	var attempts int
+	var lastTS float64
+
+	if s.statePath != "" {
+		if st, err := state.Load(s.statePath); err == nil && st != nil {
+			current = st.Phase
+			attempts = st.AttemptCount
+			if !st.UpdatedAt.IsZero() {
+				lastTS = float64(st.UpdatedAt.Unix())
+			}
+		}
+	}
+
+	sb.WriteString("# HELP kubesolo_update_phase Current update lifecycle phase (1 for active, 0 otherwise).\n")
+	sb.WriteString("# TYPE kubesolo_update_phase gauge\n")
+	for _, p := range allPhases {
+		v := 0
+		if p == current {
+			v = 1
+		}
+		sb.WriteString(fmt.Sprintf("kubesolo_update_phase{phase=%q} %d\n", string(p), v))
+	}
+
+	sb.WriteString("# HELP kubesolo_update_attempts_total Number of update attempts at the current target version.\n")
+	sb.WriteString("# TYPE kubesolo_update_attempts_total counter\n")
+	sb.WriteString(fmt.Sprintf("kubesolo_update_attempts_total %d\n", attempts))
+
+	sb.WriteString("# HELP kubesolo_update_last_attempt_timestamp_seconds Unix timestamp of the last state transition.\n")
+	sb.WriteString("# TYPE kubesolo_update_last_attempt_timestamp_seconds gauge\n")
+	sb.WriteString(fmt.Sprintf("kubesolo_update_last_attempt_timestamp_seconds %.0f\n", lastTS))
+}
+
 // readGrubenvVar reads a single variable from grubenv using simple file parse.
 func (s *Server) readGrubenvVar(key string) string {
 	data, err := os.ReadFile(s.grubenvPath)

update/pkg/metrics/metrics_test.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/portainer/kubesolo-os/update/pkg/state"
 )
 
 func TestNewServer(t *testing.T) {
@@ -247,6 +249,86 @@ func TestSafeInt(t *testing.T) {
 	}
 }
 
+func TestUpdateStateMetricsAbsentStateFile(t *testing.T) {
+	// No state path set — should emit Idle defaults so the metric series
+	// exists from first boot.
+	s := NewServer(":9100", "/tmp/nonexistent")
+
+	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	w := httptest.NewRecorder()
+	s.handleMetrics(w, req)
+
+	body, _ := io.ReadAll(w.Result().Body)
+	output := string(body)
+
+	if !strings.Contains(output, `kubesolo_update_phase{phase="idle"} 1`) {
+		t.Errorf("expected idle=1 with no state file, got:\n%s", output)
+	}
+	if !strings.Contains(output, `kubesolo_update_phase{phase="checking"} 0`) {
+		t.Errorf("expected checking=0 with no state file, got:\n%s", output)
+	}
+	if !strings.Contains(output, "kubesolo_update_attempts_total 0") {
+		t.Errorf("expected attempts=0 with no state file, got:\n%s", output)
+	}
+}
+
+func TestUpdateStateMetricsActivePhase(t *testing.T) {
+	dir := t.TempDir()
+	statePath := filepath.Join(dir, "state.json")
+
+	st := state.New()
+	if err := st.Transition(statePath, state.PhaseDownloading, "v0.3.0", ""); err != nil {
+		t.Fatalf("seed state: %v", err)
+	}
+
+	s := NewServer(":9100", "/tmp/nonexistent")
+	s.SetStatePath(statePath)
+
+	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	w := httptest.NewRecorder()
+	s.handleMetrics(w, req)
+
+	body, _ := io.ReadAll(w.Result().Body)
+	output := string(body)
+
+	if !strings.Contains(output, `kubesolo_update_phase{phase="downloading"} 1`) {
+		t.Errorf("expected downloading=1, got:\n%s", output)
+	}
+	if !strings.Contains(output, `kubesolo_update_phase{phase="idle"} 0`) {
+		t.Errorf("expected idle=0 when downloading is active, got:\n%s", output)
+	}
+	if !strings.Contains(output, "kubesolo_update_attempts_total 1") {
+		t.Errorf("expected attempts=1 after first Transition, got:\n%s", output)
+	}
+	if strings.Contains(output, "kubesolo_update_last_attempt_timestamp_seconds 0\n") {
+		t.Errorf("expected non-zero timestamp after state write, got:\n%s", output)
+	}
+}
+
+func TestUpdateStateMetricsAllPhasesEmitted(t *testing.T) {
+	// Every phase value should appear in the output, so dashboards can graph
+	// the series cleanly.
+	s := NewServer(":9100", "/tmp/nonexistent")
+
+	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	w := httptest.NewRecorder()
+	s.handleMetrics(w, req)
+
+	body, _ := io.ReadAll(w.Result().Body)
+	output := string(body)
+
+	for _, p := range []state.Phase{
+		state.PhaseIdle, state.PhaseChecking, state.PhaseDownloading,
+		state.PhaseStaged, state.PhaseActivated, state.PhaseVerifying,
+		state.PhaseSuccess, state.PhaseRolledBack, state.PhaseFailed,
+	} {
+		needle := `kubesolo_update_phase{phase="` + string(p) + `"}`
+		if !strings.Contains(output, needle) {
+			t.Errorf("phase %q not present in metrics output", p)
+		}
+	}
+}
+
 func TestReadFileString(t *testing.T) {
 	dir := t.TempDir()