Adolfo Delorenzo
bce565e2f7
Some checks failed
ARM64 Build / Build generic ARM64 disk image (push) Failing after 4s
CI / Go Tests (push) Successful in 1m31s
CI / Shellcheck (push) Successful in 47s
CI / Build Go Binaries (amd64, linux, linux-amd64) (push) Failing after 10s
CI / Build Go Binaries (arm64, linux, linux-arm64) (push) Failing after 16s
Phase 5 of v0.3. Adds an explicit, on-disk state machine to the update agent
so the lifecycle of an attempt is observable end-to-end, instead of being
inferred from logs and side effects.
New package update/pkg/state:
- Phase enum (idle, checking, downloading, staged, activated, verifying,
success, rolled_back, failed)
- UpdateState struct persisted to /var/lib/kubesolo/update/state.json
(overridable via --state). Atomic write (.tmp + rename). Survives reboots
and slot switches because the file lives on the data partition.
- Transition helper that bumps AttemptCount when an attempt starts, resets
it when the target version changes, sets/clears LastError on
failed/success transitions, and stamps StartedAt + UpdatedAt.
- 13 unit tests cover the lifecycle, atomic write, version-change reset,
error recording, idempotent SetFromVersion, garbage-file handling.
Wired into the existing commands:
- apply.go transitions Idle -> Checking -> Downloading -> Staged, with
RecordError on any step failure. Reads the active slot's version file to
populate FromVersion.
- activate.go transitions to Activated.
- healthcheck.go transitions Activated -> Verifying -> Success on pass,
or to Failed on fail. Skips transitions if state isn't post-activation
(manual healthcheck on a stable system shouldn't churn the state).
- rollback.go transitions to RolledBack with LastError="manual rollback".
- check.go intentionally untouched — checks are passive queries, not
attempts; they shouldn't reset AttemptCount.
status.go gains a --json mode that emits the full state report (A/B slots,
boot counter, full UpdateState) for orchestration tooling. Human-readable
mode also prints an Update Lifecycle section when state.phase != idle.
pkg/metrics gains three new series, derived from state.json at scrape time:
- kubesolo_update_phase{phase="..."} — 1 for current, 0 for all others;
all nine phase values always emitted so dashboards see complete series
- kubesolo_update_attempts_total
- kubesolo_update_last_attempt_timestamp_seconds
Server.SetStatePath() configures the file location; defaults to absent
which emits Idle defaults. Three new tests cover the absent / active /
all-phases-emitted cases.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
88 lines
2.8 KiB
Go
88 lines
2.8 KiB
Go
package cmd
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"github.com/portainer/kubesolo-os/update/pkg/health"
|
|
"github.com/portainer/kubesolo-os/update/pkg/state"
|
|
)
|
|
|
|
// Healthcheck performs post-boot health verification.
|
|
// If all checks pass, it marks the boot as successful in GRUB.
|
|
// This should be run after every boot (typically via a systemd unit or
|
|
// init script) to confirm the system is healthy.
|
|
//
|
|
// State transition: Activated → Verifying → Success on pass, → Failed on fail.
|
|
// If state isn't in Activated (e.g. manual run on a long-stable system), the
|
|
// state file is left alone — healthcheck still does its job.
|
|
func Healthcheck(args []string) error {
|
|
opts := parseOpts(args)
|
|
env := opts.NewBootEnv()
|
|
|
|
st, err := state.Load(opts.StatePath)
|
|
if err != nil {
|
|
slog.Warn("state file unreadable, starting fresh", "error", err)
|
|
st = state.New()
|
|
}
|
|
|
|
// Check if already marked successful
|
|
success, err := env.BootSuccess()
|
|
if err != nil {
|
|
slog.Warn("could not read boot_success", "error", err)
|
|
}
|
|
if success {
|
|
fmt.Println("Boot already marked successful")
|
|
return nil
|
|
}
|
|
|
|
// Only transition state if we're post-activation. Manual healthcheck on a
|
|
// long-stable system shouldn't reset Idle → Verifying.
|
|
postActivation := st.Phase == state.PhaseActivated
|
|
if postActivation {
|
|
if err := st.Transition(opts.StatePath, state.PhaseVerifying, "", ""); err != nil {
|
|
slog.Warn("state transition failed", "phase", state.PhaseVerifying, "error", err)
|
|
}
|
|
}
|
|
|
|
timeout := time.Duration(opts.TimeoutSecs) * time.Second
|
|
checker := health.NewChecker("", "", timeout)
|
|
|
|
slog.Info("running post-boot health checks", "timeout", timeout)
|
|
|
|
status, err := checker.WaitForHealthy()
|
|
if err != nil {
|
|
fmt.Printf("Health check FAILED: %s\n", status.Message)
|
|
fmt.Printf(" containerd: %v\n", status.Containerd)
|
|
fmt.Printf(" apiserver: %v\n", status.APIServer)
|
|
fmt.Printf(" node_ready: %v\n", status.NodeReady)
|
|
fmt.Println("\nBoot NOT marked successful — system may roll back on next reboot")
|
|
if postActivation {
|
|
_ = st.RecordError(opts.StatePath, fmt.Errorf("post-boot health check failed: %s", status.Message))
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Mark boot as successful
|
|
if err := env.MarkBootSuccess(); err != nil {
|
|
if postActivation {
|
|
_ = st.RecordError(opts.StatePath, fmt.Errorf("marking boot success: %w", err))
|
|
}
|
|
return fmt.Errorf("marking boot success: %w", err)
|
|
}
|
|
|
|
if postActivation {
|
|
if err := st.Transition(opts.StatePath, state.PhaseSuccess, "", ""); err != nil {
|
|
slog.Warn("state transition failed", "phase", state.PhaseSuccess, "error", err)
|
|
}
|
|
}
|
|
|
|
fmt.Println("Health check PASSED — boot marked successful")
|
|
fmt.Printf(" containerd: %v\n", status.Containerd)
|
|
fmt.Printf(" apiserver: %v\n", status.APIServer)
|
|
fmt.Printf(" node_ready: %v\n", status.NodeReady)
|
|
|
|
return nil
|
|
}
|