Some checks failed
ARM64 Build / Build generic ARM64 disk image (push) Failing after 4s
CI / Go Tests (push) Successful in 1m31s
CI / Shellcheck (push) Successful in 47s
CI / Build Go Binaries (amd64, linux, linux-amd64) (push) Failing after 10s
CI / Build Go Binaries (arm64, linux, linux-arm64) (push) Failing after 16s
Phase 5 of v0.3. Adds an explicit, on-disk state machine to the update agent
so the lifecycle of an attempt is observable end-to-end, instead of being
inferred from logs and side effects.
New package update/pkg/state:
- Phase enum (idle, checking, downloading, staged, activated, verifying,
success, rolled_back, failed)
- UpdateState struct persisted to /var/lib/kubesolo/update/state.json
(overridable via --state). Atomic write (.tmp + rename). Survives reboots
and slot switches because the file lives on the data partition.
- Transition helper that bumps AttemptCount when an attempt starts, resets
it when the target version changes, sets/clears LastError on
failed/success transitions, and stamps StartedAt + UpdatedAt.
- 13 unit tests cover the lifecycle, atomic write, version-change reset,
error recording, idempotent SetFromVersion, garbage-file handling.
Wired into the existing commands:
- apply.go transitions Idle -> Checking -> Downloading -> Staged, with
RecordError on any step failure. Reads the active slot's version file to
populate FromVersion.
- activate.go transitions to Activated.
- healthcheck.go transitions Activated -> Verifying -> Success on pass,
or to Failed on fail. Skips transitions if state isn't post-activation
(manual healthcheck on a stable system shouldn't churn the state).
- rollback.go transitions to RolledBack with LastError="manual rollback".
- check.go intentionally untouched — checks are passive queries, not
attempts; they shouldn't reset AttemptCount.
status.go gains a --json mode that emits the full state report (A/B slots,
boot counter, full UpdateState) for orchestration tooling. Human-readable
mode also prints an Update Lifecycle section when state.phase != idle.
pkg/metrics gains three new series, derived from state.json at scrape time:
- kubesolo_update_phase{phase="..."} — 1 for current, 0 for all others;
all nine phase values always emitted so dashboards see complete series
- kubesolo_update_attempts_total
- kubesolo_update_last_attempt_timestamp_seconds
Server.SetStatePath() configures the file location; defaults to absent
which emits Idle defaults. Three new tests cover the absent / active /
all-phases-emitted cases.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
255 lines
8.5 KiB
Go
255 lines
8.5 KiB
Go
// Package metrics exposes a lightweight Prometheus-compatible metrics endpoint
|
|
// for KubeSolo OS system and update status.
|
|
//
|
|
// Metrics exposed:
|
|
//
|
|
// kubesolo_os_info{version, active_slot} 1 (gauge, labels identify the OS)
|
|
// kubesolo_os_boot_success 1 or 0 (gauge)
|
|
// kubesolo_os_boot_counter 0-3 (gauge)
|
|
// kubesolo_os_uptime_seconds float (gauge)
|
|
// kubesolo_os_update_available 1 or 0 (gauge)
|
|
// kubesolo_os_update_last_check_timestamp_seconds unix timestamp (gauge)
|
|
// kubesolo_os_memory_total_bytes total RAM (gauge)
|
|
// kubesolo_os_memory_available_bytes available RAM (gauge)
|
|
// kubesolo_update_phase{phase} 1 for current phase, 0 for others
|
|
// kubesolo_update_attempts_total counter — attempts at current ToVersion
|
|
// kubesolo_update_last_attempt_timestamp_seconds unix timestamp of last state update
|
|
//
|
|
// This is a zero-dependency implementation — no Prometheus client library needed.
|
|
// It serves metrics in the Prometheus text exposition format.
|
|
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/portainer/kubesolo-os/update/pkg/state"
|
|
)
|
|
|
|
// Server is a lightweight Prometheus metrics HTTP server.
|
|
type Server struct {
|
|
grubenvPath string
|
|
statePath string
|
|
listenAddr string
|
|
startTime time.Time
|
|
|
|
mu sync.Mutex
|
|
updateAvailable int
|
|
lastCheckTime float64
|
|
}
|
|
|
|
// NewServer creates a new metrics server.
|
|
func NewServer(listenAddr, grubenvPath string) *Server {
|
|
return &Server{
|
|
grubenvPath: grubenvPath,
|
|
listenAddr: listenAddr,
|
|
startTime: time.Now(),
|
|
}
|
|
}
|
|
|
|
// SetStatePath sets the location of the update state.json file. If empty or
|
|
// unset, state-derived metrics are emitted with the Idle defaults.
|
|
func (s *Server) SetStatePath(p string) {
|
|
s.statePath = p
|
|
}
|
|
|
|
// allPhases lists every Phase value we emit as a kubesolo_update_phase
|
|
// time-series, so consumers see all label values (with value 0 for non-current
|
|
// phases). Mirror of validPhases in pkg/state.
|
|
var allPhases = []state.Phase{
|
|
state.PhaseIdle,
|
|
state.PhaseChecking,
|
|
state.PhaseDownloading,
|
|
state.PhaseStaged,
|
|
state.PhaseActivated,
|
|
state.PhaseVerifying,
|
|
state.PhaseSuccess,
|
|
state.PhaseRolledBack,
|
|
state.PhaseFailed,
|
|
}
|
|
|
|
// SetUpdateAvailable records whether an update is available.
|
|
func (s *Server) SetUpdateAvailable(available bool) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if available {
|
|
s.updateAvailable = 1
|
|
} else {
|
|
s.updateAvailable = 0
|
|
}
|
|
s.lastCheckTime = float64(time.Now().Unix())
|
|
}
|
|
|
|
// ListenAndServe starts the metrics HTTP server.
|
|
func (s *Server) ListenAndServe() error {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/metrics", s.handleMetrics)
|
|
mux.HandleFunc("/healthz", s.handleHealthz)
|
|
|
|
slog.Info("starting metrics server", "addr", s.listenAddr)
|
|
return http.ListenAndServe(s.listenAddr, mux)
|
|
}
|
|
|
|
func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
fmt.Fprint(w, "ok\n")
|
|
}
|
|
|
|
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
|
|
|
var sb strings.Builder
|
|
|
|
// OS info
|
|
version := readFileString("/etc/kubesolo-os-version")
|
|
activeSlot := s.readGrubenvVar("active_slot")
|
|
sb.WriteString("# HELP kubesolo_os_info KubeSolo OS version and slot info.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_info gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_info{version=%q,active_slot=%q} 1\n",
|
|
version, activeSlot))
|
|
|
|
// Boot status
|
|
bootSuccess := s.readGrubenvVar("boot_success")
|
|
bootCounter := s.readGrubenvVar("boot_counter")
|
|
sb.WriteString("# HELP kubesolo_os_boot_success Whether the current boot was marked successful.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_boot_success gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_boot_success %s\n", safeInt(bootSuccess, "0")))
|
|
sb.WriteString("# HELP kubesolo_os_boot_counter Remaining boot attempts before rollback.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_boot_counter gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_boot_counter %s\n", safeInt(bootCounter, "0")))
|
|
|
|
// Uptime
|
|
uptime := time.Since(s.startTime).Seconds()
|
|
sb.WriteString("# HELP kubesolo_os_uptime_seconds Time since the metrics server started.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_uptime_seconds gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_uptime_seconds %.1f\n", uptime))
|
|
|
|
// Update status
|
|
s.mu.Lock()
|
|
updateAvail := s.updateAvailable
|
|
lastCheck := s.lastCheckTime
|
|
s.mu.Unlock()
|
|
|
|
sb.WriteString("# HELP kubesolo_os_update_available Whether an OS update is available.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_update_available gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_update_available %d\n", updateAvail))
|
|
sb.WriteString("# HELP kubesolo_os_update_last_check_timestamp_seconds Unix timestamp of last update check.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_update_last_check_timestamp_seconds gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_update_last_check_timestamp_seconds %.0f\n", lastCheck))
|
|
|
|
// Memory
|
|
memTotal, memAvail := readMemInfo()
|
|
sb.WriteString("# HELP kubesolo_os_memory_total_bytes Total system memory in bytes.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_memory_total_bytes gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_memory_total_bytes %d\n", memTotal))
|
|
sb.WriteString("# HELP kubesolo_os_memory_available_bytes Available system memory in bytes.\n")
|
|
sb.WriteString("# TYPE kubesolo_os_memory_available_bytes gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_os_memory_available_bytes %d\n", memAvail))
|
|
|
|
// Update lifecycle (from state.json)
|
|
s.writeUpdateStateMetrics(&sb)
|
|
|
|
fmt.Fprint(w, sb.String())
|
|
}
|
|
|
|
// writeUpdateStateMetrics appends update-lifecycle metrics derived from the
|
|
// state.json file. If the file is missing or unreadable, emits the Idle
|
|
// defaults so the metric series exists at all times.
|
|
func (s *Server) writeUpdateStateMetrics(sb *strings.Builder) {
|
|
current := state.PhaseIdle
|
|
var attempts int
|
|
var lastTS float64
|
|
|
|
if s.statePath != "" {
|
|
if st, err := state.Load(s.statePath); err == nil && st != nil {
|
|
current = st.Phase
|
|
attempts = st.AttemptCount
|
|
if !st.UpdatedAt.IsZero() {
|
|
lastTS = float64(st.UpdatedAt.Unix())
|
|
}
|
|
}
|
|
}
|
|
|
|
sb.WriteString("# HELP kubesolo_update_phase Current update lifecycle phase (1 for active, 0 otherwise).\n")
|
|
sb.WriteString("# TYPE kubesolo_update_phase gauge\n")
|
|
for _, p := range allPhases {
|
|
v := 0
|
|
if p == current {
|
|
v = 1
|
|
}
|
|
sb.WriteString(fmt.Sprintf("kubesolo_update_phase{phase=%q} %d\n", string(p), v))
|
|
}
|
|
|
|
sb.WriteString("# HELP kubesolo_update_attempts_total Number of update attempts at the current target version.\n")
|
|
sb.WriteString("# TYPE kubesolo_update_attempts_total counter\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_update_attempts_total %d\n", attempts))
|
|
|
|
sb.WriteString("# HELP kubesolo_update_last_attempt_timestamp_seconds Unix timestamp of the last state transition.\n")
|
|
sb.WriteString("# TYPE kubesolo_update_last_attempt_timestamp_seconds gauge\n")
|
|
sb.WriteString(fmt.Sprintf("kubesolo_update_last_attempt_timestamp_seconds %.0f\n", lastTS))
|
|
}
|
|
|
|
// readGrubenvVar reads a single variable from grubenv using simple file parse.
|
|
func (s *Server) readGrubenvVar(key string) string {
|
|
data, err := os.ReadFile(s.grubenvPath)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
parts := strings.SplitN(line, "=", 2)
|
|
if len(parts) == 2 && strings.TrimSpace(parts[0]) == key {
|
|
return strings.TrimSpace(parts[1])
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// readFileString reads a file and returns trimmed content.
|
|
func readFileString(path string) string {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
return strings.TrimSpace(string(data))
|
|
}
|
|
|
|
// readMemInfo parses /proc/meminfo for total and available memory.
|
|
func readMemInfo() (total, available int64) {
|
|
data, err := os.ReadFile("/proc/meminfo")
|
|
if err != nil {
|
|
return 0, 0
|
|
}
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 2 {
|
|
continue
|
|
}
|
|
val, err := strconv.ParseInt(fields[1], 10, 64)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
// /proc/meminfo values are in kB
|
|
switch fields[0] {
|
|
case "MemTotal:":
|
|
total = val * 1024
|
|
case "MemAvailable:":
|
|
available = val * 1024
|
|
}
|
|
}
|
|
return total, available
|
|
}
|
|
|
|
// safeInt returns the value if it's a valid integer, otherwise the default.
|
|
func safeInt(s, def string) string {
|
|
if _, err := strconv.Atoi(s); err != nil {
|
|
return def
|
|
}
|
|
return s
|
|
}
|