Files
kubesolo-os/update/pkg/metrics/metrics_test.go
Adolfo Delorenzo bce565e2f7
Some checks failed
ARM64 Build / Build generic ARM64 disk image (push) Failing after 4s
CI / Go Tests (push) Successful in 1m31s
CI / Shellcheck (push) Successful in 47s
CI / Build Go Binaries (amd64, linux, linux-amd64) (push) Failing after 10s
CI / Build Go Binaries (arm64, linux, linux-arm64) (push) Failing after 16s
feat(update): persistent state machine + lifecycle metrics
Phase 5 of v0.3. Adds an explicit, on-disk state machine to the update agent
so the lifecycle of an attempt is observable end-to-end, instead of being
inferred from logs and side effects.

New package update/pkg/state:
- Phase enum (idle, checking, downloading, staged, activated, verifying,
  success, rolled_back, failed)
- UpdateState struct persisted to /var/lib/kubesolo/update/state.json
  (overridable via --state). Atomic write (.tmp + rename). Survives reboots
  and slot switches because the file lives on the data partition.
- Transition helper that bumps AttemptCount when an attempt starts, resets
  it when the target version changes, sets/clears LastError on
  failed/success transitions, and stamps StartedAt + UpdatedAt.
- 13 unit tests cover the lifecycle, atomic write, version-change reset,
  error recording, idempotent SetFromVersion, garbage-file handling.

Wired into the existing commands:
- apply.go transitions Idle -> Checking -> Downloading -> Staged, with
  RecordError on any step failure. Reads the active slot's version file to
  populate FromVersion.
- activate.go transitions to Activated.
- healthcheck.go transitions Activated -> Verifying -> Success on pass,
  or to Failed on fail. Skips transitions if state isn't post-activation
  (manual healthcheck on a stable system shouldn't churn the state).
- rollback.go transitions to RolledBack with LastError="manual rollback".
- check.go intentionally untouched — checks are passive queries, not
  attempts; they shouldn't reset AttemptCount.

status.go gains a --json mode that emits the full state report (A/B slots,
boot counter, full UpdateState) for orchestration tooling. Human-readable
mode also prints an Update Lifecycle section when state.phase != idle.

pkg/metrics gains three new series, derived from state.json at scrape time:
- kubesolo_update_phase{phase="..."} — 1 for current, 0 for all others;
  all nine phase values always emitted so dashboards see complete series
- kubesolo_update_attempts_total
- kubesolo_update_last_attempt_timestamp_seconds
Server.SetStatePath() configures the file location; defaults to absent
which emits Idle defaults. Three new tests cover the absent / active /
all-phases-emitted cases.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-14 18:11:47 -06:00

351 lines
9.2 KiB
Go

package metrics
import (
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/portainer/kubesolo-os/update/pkg/state"
)
func TestNewServer(t *testing.T) {
s := NewServer(":9100", "/boot/grub/grubenv")
if s == nil {
t.Fatal("NewServer returned nil")
}
if s.listenAddr != ":9100" {
t.Errorf("listenAddr = %q, want %q", s.listenAddr, ":9100")
}
if s.grubenvPath != "/boot/grub/grubenv" {
t.Errorf("grubenvPath = %q, want %q", s.grubenvPath, "/boot/grub/grubenv")
}
if s.startTime.IsZero() {
t.Error("startTime not set")
}
}
func TestSetUpdateAvailable(t *testing.T) {
s := NewServer(":9100", "/tmp/nonexistent")
s.SetUpdateAvailable(true)
s.mu.Lock()
if s.updateAvailable != 1 {
t.Errorf("updateAvailable = %d, want 1", s.updateAvailable)
}
if s.lastCheckTime == 0 {
t.Error("lastCheckTime not updated")
}
s.mu.Unlock()
s.SetUpdateAvailable(false)
s.mu.Lock()
if s.updateAvailable != 0 {
t.Errorf("updateAvailable = %d, want 0", s.updateAvailable)
}
s.mu.Unlock()
}
func TestHandleHealthz(t *testing.T) {
s := NewServer(":9100", "/tmp/nonexistent")
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
w := httptest.NewRecorder()
s.handleHealthz(w, req)
resp := w.Result()
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
body, _ := io.ReadAll(resp.Body)
if string(body) != "ok\n" {
t.Errorf("body = %q, want %q", string(body), "ok\n")
}
}
func TestHandleMetrics(t *testing.T) {
// Create a temp grubenv
dir := t.TempDir()
grubenv := filepath.Join(dir, "grubenv")
content := "active_slot=A\nboot_success=1\nboot_counter=3\n"
if err := os.WriteFile(grubenv, []byte(content), 0644); err != nil {
t.Fatal(err)
}
// Create a fake version file — we'll test that missing version returns "unknown"
s := NewServer(":9100", grubenv)
s.SetUpdateAvailable(true)
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
w := httptest.NewRecorder()
s.handleMetrics(w, req)
resp := w.Result()
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
ct := resp.Header.Get("Content-Type")
if !strings.Contains(ct, "text/plain") {
t.Errorf("Content-Type = %q, want text/plain", ct)
}
body, _ := io.ReadAll(resp.Body)
output := string(body)
// Check expected metrics are present
expectedMetrics := []string{
"kubesolo_os_info{",
"active_slot=\"A\"",
"kubesolo_os_boot_success 1",
"kubesolo_os_boot_counter 3",
"kubesolo_os_uptime_seconds",
"kubesolo_os_update_available 1",
"kubesolo_os_update_last_check_timestamp_seconds",
"kubesolo_os_memory_total_bytes",
"kubesolo_os_memory_available_bytes",
}
for _, expected := range expectedMetrics {
if !strings.Contains(output, expected) {
t.Errorf("metrics output missing %q\nfull output:\n%s", expected, output)
}
}
// Check HELP and TYPE comments
expectedHelp := []string{
"# HELP kubesolo_os_info",
"# TYPE kubesolo_os_info gauge",
"# HELP kubesolo_os_boot_success",
"# HELP kubesolo_os_uptime_seconds",
"# HELP kubesolo_os_update_available",
"# HELP kubesolo_os_memory_total_bytes",
}
for _, expected := range expectedHelp {
if !strings.Contains(output, expected) {
t.Errorf("metrics output missing %q", expected)
}
}
}
func TestHandleMetricsMissingGrubenv(t *testing.T) {
s := NewServer(":9100", "/tmp/nonexistent-grubenv-file")
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
w := httptest.NewRecorder()
s.handleMetrics(w, req)
resp := w.Result()
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
output := string(body)
// Should still render with defaults
if !strings.Contains(output, "kubesolo_os_boot_success 0") {
t.Errorf("expected boot_success=0 with missing grubenv, got:\n%s", output)
}
if !strings.Contains(output, "kubesolo_os_boot_counter 0") {
t.Errorf("expected boot_counter=0 with missing grubenv, got:\n%s", output)
}
// active_slot should be empty
if !strings.Contains(output, `active_slot=""`) {
t.Errorf("expected empty active_slot with missing grubenv, got:\n%s", output)
}
}
func TestHandleMetricsUpdateNotAvailable(t *testing.T) {
s := NewServer(":9100", "/tmp/nonexistent")
// Don't call SetUpdateAvailable — should default to 0
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
w := httptest.NewRecorder()
s.handleMetrics(w, req)
resp := w.Result()
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
output := string(body)
if !strings.Contains(output, "kubesolo_os_update_available 0") {
t.Errorf("expected update_available=0 by default, got:\n%s", output)
}
if !strings.Contains(output, "kubesolo_os_update_last_check_timestamp_seconds 0") {
t.Errorf("expected last_check=0 by default, got:\n%s", output)
}
}
func TestReadGrubenvVar(t *testing.T) {
dir := t.TempDir()
grubenv := filepath.Join(dir, "grubenv")
content := "active_slot=B\nboot_success=0\nboot_counter=2\nsome_other=value\n"
if err := os.WriteFile(grubenv, []byte(content), 0644); err != nil {
t.Fatal(err)
}
s := NewServer(":9100", grubenv)
tests := []struct {
key string
want string
}{
{"active_slot", "B"},
{"boot_success", "0"},
{"boot_counter", "2"},
{"some_other", "value"},
{"nonexistent", ""},
}
for _, tt := range tests {
got := s.readGrubenvVar(tt.key)
if got != tt.want {
t.Errorf("readGrubenvVar(%q) = %q, want %q", tt.key, got, tt.want)
}
}
}
func TestReadGrubenvVarMissingFile(t *testing.T) {
s := NewServer(":9100", "/tmp/nonexistent-grubenv")
got := s.readGrubenvVar("active_slot")
if got != "" {
t.Errorf("readGrubenvVar with missing file = %q, want empty", got)
}
}
func TestSafeInt(t *testing.T) {
tests := []struct {
input string
def string
want string
}{
{"42", "0", "42"},
{"0", "0", "0"},
{"3", "0", "3"},
{"", "0", "0"},
{"abc", "0", "0"},
{"1.5", "0", "0"},
{"-1", "0", "-1"},
}
for _, tt := range tests {
got := safeInt(tt.input, tt.def)
if got != tt.want {
t.Errorf("safeInt(%q, %q) = %q, want %q", tt.input, tt.def, got, tt.want)
}
}
}
func TestUpdateStateMetricsAbsentStateFile(t *testing.T) {
// No state path set — should emit Idle defaults so the metric series
// exists from first boot.
s := NewServer(":9100", "/tmp/nonexistent")
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
w := httptest.NewRecorder()
s.handleMetrics(w, req)
body, _ := io.ReadAll(w.Result().Body)
output := string(body)
if !strings.Contains(output, `kubesolo_update_phase{phase="idle"} 1`) {
t.Errorf("expected idle=1 with no state file, got:\n%s", output)
}
if !strings.Contains(output, `kubesolo_update_phase{phase="checking"} 0`) {
t.Errorf("expected checking=0 with no state file, got:\n%s", output)
}
if !strings.Contains(output, "kubesolo_update_attempts_total 0") {
t.Errorf("expected attempts=0 with no state file, got:\n%s", output)
}
}
func TestUpdateStateMetricsActivePhase(t *testing.T) {
dir := t.TempDir()
statePath := filepath.Join(dir, "state.json")
st := state.New()
if err := st.Transition(statePath, state.PhaseDownloading, "v0.3.0", ""); err != nil {
t.Fatalf("seed state: %v", err)
}
s := NewServer(":9100", "/tmp/nonexistent")
s.SetStatePath(statePath)
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
w := httptest.NewRecorder()
s.handleMetrics(w, req)
body, _ := io.ReadAll(w.Result().Body)
output := string(body)
if !strings.Contains(output, `kubesolo_update_phase{phase="downloading"} 1`) {
t.Errorf("expected downloading=1, got:\n%s", output)
}
if !strings.Contains(output, `kubesolo_update_phase{phase="idle"} 0`) {
t.Errorf("expected idle=0 when downloading is active, got:\n%s", output)
}
if !strings.Contains(output, "kubesolo_update_attempts_total 1") {
t.Errorf("expected attempts=1 after first Transition, got:\n%s", output)
}
if strings.Contains(output, "kubesolo_update_last_attempt_timestamp_seconds 0\n") {
t.Errorf("expected non-zero timestamp after state write, got:\n%s", output)
}
}
func TestUpdateStateMetricsAllPhasesEmitted(t *testing.T) {
// Every phase value should appear in the output, so dashboards can graph
// the series cleanly.
s := NewServer(":9100", "/tmp/nonexistent")
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
w := httptest.NewRecorder()
s.handleMetrics(w, req)
body, _ := io.ReadAll(w.Result().Body)
output := string(body)
for _, p := range []state.Phase{
state.PhaseIdle, state.PhaseChecking, state.PhaseDownloading,
state.PhaseStaged, state.PhaseActivated, state.PhaseVerifying,
state.PhaseSuccess, state.PhaseRolledBack, state.PhaseFailed,
} {
needle := `kubesolo_update_phase{phase="` + string(p) + `"}`
if !strings.Contains(output, needle) {
t.Errorf("phase %q not present in metrics output", p)
}
}
}
func TestReadFileString(t *testing.T) {
dir := t.TempDir()
// Test existing file
path := filepath.Join(dir, "version")
if err := os.WriteFile(path, []byte(" 1.2.3\n "), 0644); err != nil {
t.Fatal(err)
}
got := readFileString(path)
if got != "1.2.3" {
t.Errorf("readFileString = %q, want %q", got, "1.2.3")
}
// Test missing file
got = readFileString("/tmp/nonexistent-file-12345")
if got != "unknown" {
t.Errorf("readFileString missing file = %q, want %q", got, "unknown")
}
}