feat(update): persistent state machine + lifecycle metrics
Some checks failed
ARM64 Build / Build generic ARM64 disk image (push) Failing after 4s
CI / Go Tests (push) Successful in 1m31s
CI / Shellcheck (push) Successful in 47s
CI / Build Go Binaries (amd64, linux, linux-amd64) (push) Failing after 10s
CI / Build Go Binaries (arm64, linux, linux-arm64) (push) Failing after 16s
Some checks failed
ARM64 Build / Build generic ARM64 disk image (push) Failing after 4s
CI / Go Tests (push) Successful in 1m31s
CI / Shellcheck (push) Successful in 47s
CI / Build Go Binaries (amd64, linux, linux-amd64) (push) Failing after 10s
CI / Build Go Binaries (arm64, linux, linux-arm64) (push) Failing after 16s
Phase 5 of v0.3. Adds an explicit, on-disk state machine to the update agent
so the lifecycle of an attempt is observable end-to-end, instead of being
inferred from logs and side effects.
New package update/pkg/state:
- Phase enum (idle, checking, downloading, staged, activated, verifying,
success, rolled_back, failed)
- UpdateState struct persisted to /var/lib/kubesolo/update/state.json
(overridable via --state). Atomic write (.tmp + rename). Survives reboots
and slot switches because the file lives on the data partition.
- Transition helper that bumps AttemptCount when an attempt starts, resets
it when the target version changes, sets/clears LastError on
failed/success transitions, and stamps StartedAt + UpdatedAt.
- 13 unit tests cover the lifecycle, atomic write, version-change reset,
error recording, idempotent SetFromVersion, garbage-file handling.
Wired into the existing commands:
- apply.go transitions Idle -> Checking -> Downloading -> Staged, with
RecordError on any step failure. Reads the active slot's version file to
populate FromVersion.
- activate.go transitions to Activated.
- healthcheck.go transitions Activated -> Verifying -> Success on pass,
or to Failed on fail. Skips transitions if state isn't post-activation
(manual healthcheck on a stable system shouldn't churn the state).
- rollback.go transitions to RolledBack with LastError="manual rollback".
- check.go intentionally untouched — checks are passive queries, not
attempts; they shouldn't reset AttemptCount.
status.go gains a --json mode that emits the full state report (A/B slots,
boot counter, full UpdateState) for orchestration tooling. Human-readable
mode also prints an Update Lifecycle section when state.phase != idle.
pkg/metrics gains three new series, derived from state.json at scrape time:
- kubesolo_update_phase{phase="..."} — 1 for current, 0 for all others;
all nine phase values always emitted so dashboards see complete series
- kubesolo_update_attempts_total
- kubesolo_update_last_attempt_timestamp_seconds
Server.SetStatePath() configures the file location; defaults to absent
which emits Idle defaults. Three new tests cover the absent / active /
all-phases-emitted cases.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/portainer/kubesolo-os/update/pkg/state"
|
||||
)
|
||||
|
||||
func TestNewServer(t *testing.T) {
|
||||
@@ -247,6 +249,86 @@ func TestSafeInt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateStateMetricsAbsentStateFile(t *testing.T) {
|
||||
// No state path set — should emit Idle defaults so the metric series
|
||||
// exists from first boot.
|
||||
s := NewServer(":9100", "/tmp/nonexistent")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
w := httptest.NewRecorder()
|
||||
s.handleMetrics(w, req)
|
||||
|
||||
body, _ := io.ReadAll(w.Result().Body)
|
||||
output := string(body)
|
||||
|
||||
if !strings.Contains(output, `kubesolo_update_phase{phase="idle"} 1`) {
|
||||
t.Errorf("expected idle=1 with no state file, got:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(output, `kubesolo_update_phase{phase="checking"} 0`) {
|
||||
t.Errorf("expected checking=0 with no state file, got:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(output, "kubesolo_update_attempts_total 0") {
|
||||
t.Errorf("expected attempts=0 with no state file, got:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateStateMetricsActivePhase(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
statePath := filepath.Join(dir, "state.json")
|
||||
|
||||
st := state.New()
|
||||
if err := st.Transition(statePath, state.PhaseDownloading, "v0.3.0", ""); err != nil {
|
||||
t.Fatalf("seed state: %v", err)
|
||||
}
|
||||
|
||||
s := NewServer(":9100", "/tmp/nonexistent")
|
||||
s.SetStatePath(statePath)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
w := httptest.NewRecorder()
|
||||
s.handleMetrics(w, req)
|
||||
|
||||
body, _ := io.ReadAll(w.Result().Body)
|
||||
output := string(body)
|
||||
|
||||
if !strings.Contains(output, `kubesolo_update_phase{phase="downloading"} 1`) {
|
||||
t.Errorf("expected downloading=1, got:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(output, `kubesolo_update_phase{phase="idle"} 0`) {
|
||||
t.Errorf("expected idle=0 when downloading is active, got:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(output, "kubesolo_update_attempts_total 1") {
|
||||
t.Errorf("expected attempts=1 after first Transition, got:\n%s", output)
|
||||
}
|
||||
if strings.Contains(output, "kubesolo_update_last_attempt_timestamp_seconds 0\n") {
|
||||
t.Errorf("expected non-zero timestamp after state write, got:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateStateMetricsAllPhasesEmitted(t *testing.T) {
|
||||
// Every phase value should appear in the output, so dashboards can graph
|
||||
// the series cleanly.
|
||||
s := NewServer(":9100", "/tmp/nonexistent")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
w := httptest.NewRecorder()
|
||||
s.handleMetrics(w, req)
|
||||
|
||||
body, _ := io.ReadAll(w.Result().Body)
|
||||
output := string(body)
|
||||
|
||||
for _, p := range []state.Phase{
|
||||
state.PhaseIdle, state.PhaseChecking, state.PhaseDownloading,
|
||||
state.PhaseStaged, state.PhaseActivated, state.PhaseVerifying,
|
||||
state.PhaseSuccess, state.PhaseRolledBack, state.PhaseFailed,
|
||||
} {
|
||||
needle := `kubesolo_update_phase{phase="` + string(p) + `"}`
|
||||
if !strings.Contains(output, needle) {
|
||||
t.Errorf("phase %q not present in metrics output", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFileString(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user