feat: add distribution and fleet management — CI/CD, OCI, metrics, ARM64 (Phase 5)
- Gitea Actions CI pipeline: Go tests, build, shellcheck on push/PR - Gitea Actions release pipeline: full build + artifact upload on version tags - OCI container image builder for registry-based OS distribution - Zero-dependency Prometheus metrics endpoint (kubesolo_os_info, boot, memory, update status) with 10 tests - USB provisioning tool for air-gapped deployments with cloud-init injection - ARM64 cross-compilation support (TARGET_ARCH env var + build-cross.sh) - Updated build scripts to accept TARGET_ARCH for both amd64 and arm64 - New Makefile targets: oci-image, build-cross Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
21
update/cmd/metrics.go
Normal file
21
update/cmd/metrics.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
|
||||
"github.com/portainer/kubesolo-os/update/pkg/metrics"
|
||||
)
|
||||
|
||||
// Metrics starts the Prometheus-compatible metrics HTTP server.
|
||||
func Metrics(args []string) error {
|
||||
fs := flag.NewFlagSet("metrics", flag.ExitOnError)
|
||||
listenAddr := fs.String("listen", ":9100", "Metrics HTTP listen address")
|
||||
grubenvPath := fs.String("grubenv", "/boot/grub/grubenv", "Path to grubenv file")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("parse flags: %w", err)
|
||||
}
|
||||
|
||||
srv := metrics.NewServer(*listenAddr, *grubenvPath)
|
||||
return srv.ListenAndServe()
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
// kubesolo-update status Show current A/B slot and boot status
|
||||
// kubesolo-update sign Sign update artifacts with Ed25519 key
|
||||
// kubesolo-update genkey Generate new Ed25519 signing key pair
|
||||
// kubesolo-update metrics Start Prometheus-compatible metrics server
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -48,6 +49,8 @@ func main() {
|
||||
err = cmd.Sign(os.Args[2:])
|
||||
case "genkey":
|
||||
err = cmd.GenKey(os.Args[2:])
|
||||
case "metrics":
|
||||
err = cmd.Metrics(os.Args[2:])
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown command: %s\n\n", os.Args[1])
|
||||
usage()
|
||||
@@ -72,6 +75,7 @@ Commands:
|
||||
status Show current A/B slot and boot status
|
||||
sign Sign artifacts with Ed25519 private key (build system)
|
||||
genkey Generate new Ed25519 signing key pair
|
||||
metrics Start Prometheus-compatible metrics HTTP server
|
||||
|
||||
Options:
|
||||
--server URL Update server URL (default: from /etc/kubesolo/update.conf)
|
||||
|
||||
187
update/pkg/metrics/metrics.go
Normal file
187
update/pkg/metrics/metrics.go
Normal file
@@ -0,0 +1,187 @@
|
||||
// Package metrics exposes a lightweight Prometheus-compatible metrics endpoint
|
||||
// for KubeSolo OS system and update status.
|
||||
//
|
||||
// Metrics exposed:
|
||||
//
|
||||
// kubesolo_os_info{version, active_slot} 1 (gauge, labels identify the OS)
|
||||
// kubesolo_os_boot_success 1 or 0 (gauge)
|
||||
// kubesolo_os_boot_counter 0-3 (gauge)
|
||||
// kubesolo_os_uptime_seconds float (gauge)
|
||||
// kubesolo_os_update_available 1 or 0 (gauge)
|
||||
// kubesolo_os_update_last_check_timestamp_seconds unix timestamp (gauge)
|
||||
// kubesolo_os_memory_total_bytes total RAM (gauge)
|
||||
// kubesolo_os_memory_available_bytes available RAM (gauge)
|
||||
//
|
||||
// This is a zero-dependency implementation — no Prometheus client library needed.
|
||||
// It serves metrics in the Prometheus text exposition format.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Server is a lightweight Prometheus metrics HTTP server.
|
||||
type Server struct {
|
||||
grubenvPath string
|
||||
listenAddr string
|
||||
startTime time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
updateAvailable int
|
||||
lastCheckTime float64
|
||||
}
|
||||
|
||||
// NewServer creates a new metrics server.
|
||||
func NewServer(listenAddr, grubenvPath string) *Server {
|
||||
return &Server{
|
||||
grubenvPath: grubenvPath,
|
||||
listenAddr: listenAddr,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// SetUpdateAvailable records whether an update is available.
|
||||
func (s *Server) SetUpdateAvailable(available bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if available {
|
||||
s.updateAvailable = 1
|
||||
} else {
|
||||
s.updateAvailable = 0
|
||||
}
|
||||
s.lastCheckTime = float64(time.Now().Unix())
|
||||
}
|
||||
|
||||
// ListenAndServe starts the metrics HTTP server.
|
||||
func (s *Server) ListenAndServe() error {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/metrics", s.handleMetrics)
|
||||
mux.HandleFunc("/healthz", s.handleHealthz)
|
||||
|
||||
slog.Info("starting metrics server", "addr", s.listenAddr)
|
||||
return http.ListenAndServe(s.listenAddr, mux)
|
||||
}
|
||||
|
||||
func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, "ok\n")
|
||||
}
|
||||
|
||||
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
|
||||
var sb strings.Builder
|
||||
|
||||
// OS info
|
||||
version := readFileString("/etc/kubesolo-os-version")
|
||||
activeSlot := s.readGrubenvVar("active_slot")
|
||||
sb.WriteString("# HELP kubesolo_os_info KubeSolo OS version and slot info.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_info gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_info{version=%q,active_slot=%q} 1\n",
|
||||
version, activeSlot))
|
||||
|
||||
// Boot status
|
||||
bootSuccess := s.readGrubenvVar("boot_success")
|
||||
bootCounter := s.readGrubenvVar("boot_counter")
|
||||
sb.WriteString("# HELP kubesolo_os_boot_success Whether the current boot was marked successful.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_boot_success gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_boot_success %s\n", safeInt(bootSuccess, "0")))
|
||||
sb.WriteString("# HELP kubesolo_os_boot_counter Remaining boot attempts before rollback.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_boot_counter gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_boot_counter %s\n", safeInt(bootCounter, "0")))
|
||||
|
||||
// Uptime
|
||||
uptime := time.Since(s.startTime).Seconds()
|
||||
sb.WriteString("# HELP kubesolo_os_uptime_seconds Time since the metrics server started.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_uptime_seconds gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_uptime_seconds %.1f\n", uptime))
|
||||
|
||||
// Update status
|
||||
s.mu.Lock()
|
||||
updateAvail := s.updateAvailable
|
||||
lastCheck := s.lastCheckTime
|
||||
s.mu.Unlock()
|
||||
|
||||
sb.WriteString("# HELP kubesolo_os_update_available Whether an OS update is available.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_update_available gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_update_available %d\n", updateAvail))
|
||||
sb.WriteString("# HELP kubesolo_os_update_last_check_timestamp_seconds Unix timestamp of last update check.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_update_last_check_timestamp_seconds gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_update_last_check_timestamp_seconds %.0f\n", lastCheck))
|
||||
|
||||
// Memory
|
||||
memTotal, memAvail := readMemInfo()
|
||||
sb.WriteString("# HELP kubesolo_os_memory_total_bytes Total system memory in bytes.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_memory_total_bytes gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_memory_total_bytes %d\n", memTotal))
|
||||
sb.WriteString("# HELP kubesolo_os_memory_available_bytes Available system memory in bytes.\n")
|
||||
sb.WriteString("# TYPE kubesolo_os_memory_available_bytes gauge\n")
|
||||
sb.WriteString(fmt.Sprintf("kubesolo_os_memory_available_bytes %d\n", memAvail))
|
||||
|
||||
fmt.Fprint(w, sb.String())
|
||||
}
|
||||
|
||||
// readGrubenvVar reads a single variable from grubenv using simple file parse.
|
||||
func (s *Server) readGrubenvVar(key string) string {
|
||||
data, err := os.ReadFile(s.grubenvPath)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
parts := strings.SplitN(line, "=", 2)
|
||||
if len(parts) == 2 && strings.TrimSpace(parts[0]) == key {
|
||||
return strings.TrimSpace(parts[1])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// readFileString reads a file and returns trimmed content.
|
||||
func readFileString(path string) string {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
return strings.TrimSpace(string(data))
|
||||
}
|
||||
|
||||
// readMemInfo parses /proc/meminfo for total and available memory.
|
||||
func readMemInfo() (total, available int64) {
|
||||
data, err := os.ReadFile("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
val, err := strconv.ParseInt(fields[1], 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// /proc/meminfo values are in kB
|
||||
switch fields[0] {
|
||||
case "MemTotal:":
|
||||
total = val * 1024
|
||||
case "MemAvailable:":
|
||||
available = val * 1024
|
||||
}
|
||||
}
|
||||
return total, available
|
||||
}
|
||||
|
||||
// safeInt returns the value if it's a valid integer, otherwise the default.
|
||||
func safeInt(s, def string) string {
|
||||
if _, err := strconv.Atoi(s); err != nil {
|
||||
return def
|
||||
}
|
||||
return s
|
||||
}
|
||||
268
update/pkg/metrics/metrics_test.go
Normal file
268
update/pkg/metrics/metrics_test.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewServer(t *testing.T) {
|
||||
s := NewServer(":9100", "/boot/grub/grubenv")
|
||||
if s == nil {
|
||||
t.Fatal("NewServer returned nil")
|
||||
}
|
||||
if s.listenAddr != ":9100" {
|
||||
t.Errorf("listenAddr = %q, want %q", s.listenAddr, ":9100")
|
||||
}
|
||||
if s.grubenvPath != "/boot/grub/grubenv" {
|
||||
t.Errorf("grubenvPath = %q, want %q", s.grubenvPath, "/boot/grub/grubenv")
|
||||
}
|
||||
if s.startTime.IsZero() {
|
||||
t.Error("startTime not set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetUpdateAvailable(t *testing.T) {
|
||||
s := NewServer(":9100", "/tmp/nonexistent")
|
||||
|
||||
s.SetUpdateAvailable(true)
|
||||
s.mu.Lock()
|
||||
if s.updateAvailable != 1 {
|
||||
t.Errorf("updateAvailable = %d, want 1", s.updateAvailable)
|
||||
}
|
||||
if s.lastCheckTime == 0 {
|
||||
t.Error("lastCheckTime not updated")
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.SetUpdateAvailable(false)
|
||||
s.mu.Lock()
|
||||
if s.updateAvailable != 0 {
|
||||
t.Errorf("updateAvailable = %d, want 0", s.updateAvailable)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestHandleHealthz(t *testing.T) {
|
||||
s := NewServer(":9100", "/tmp/nonexistent")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
s.handleHealthz(w, req)
|
||||
|
||||
resp := w.Result()
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if string(body) != "ok\n" {
|
||||
t.Errorf("body = %q, want %q", string(body), "ok\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleMetrics(t *testing.T) {
|
||||
// Create a temp grubenv
|
||||
dir := t.TempDir()
|
||||
grubenv := filepath.Join(dir, "grubenv")
|
||||
content := "active_slot=A\nboot_success=1\nboot_counter=3\n"
|
||||
if err := os.WriteFile(grubenv, []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create a fake version file — we'll test that missing version returns "unknown"
|
||||
s := NewServer(":9100", grubenv)
|
||||
s.SetUpdateAvailable(true)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
s.handleMetrics(w, req)
|
||||
|
||||
resp := w.Result()
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
if !strings.Contains(ct, "text/plain") {
|
||||
t.Errorf("Content-Type = %q, want text/plain", ct)
|
||||
}
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
output := string(body)
|
||||
|
||||
// Check expected metrics are present
|
||||
expectedMetrics := []string{
|
||||
"kubesolo_os_info{",
|
||||
"active_slot=\"A\"",
|
||||
"kubesolo_os_boot_success 1",
|
||||
"kubesolo_os_boot_counter 3",
|
||||
"kubesolo_os_uptime_seconds",
|
||||
"kubesolo_os_update_available 1",
|
||||
"kubesolo_os_update_last_check_timestamp_seconds",
|
||||
"kubesolo_os_memory_total_bytes",
|
||||
"kubesolo_os_memory_available_bytes",
|
||||
}
|
||||
|
||||
for _, expected := range expectedMetrics {
|
||||
if !strings.Contains(output, expected) {
|
||||
t.Errorf("metrics output missing %q\nfull output:\n%s", expected, output)
|
||||
}
|
||||
}
|
||||
|
||||
// Check HELP and TYPE comments
|
||||
expectedHelp := []string{
|
||||
"# HELP kubesolo_os_info",
|
||||
"# TYPE kubesolo_os_info gauge",
|
||||
"# HELP kubesolo_os_boot_success",
|
||||
"# HELP kubesolo_os_uptime_seconds",
|
||||
"# HELP kubesolo_os_update_available",
|
||||
"# HELP kubesolo_os_memory_total_bytes",
|
||||
}
|
||||
|
||||
for _, expected := range expectedHelp {
|
||||
if !strings.Contains(output, expected) {
|
||||
t.Errorf("metrics output missing %q", expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleMetricsMissingGrubenv(t *testing.T) {
|
||||
s := NewServer(":9100", "/tmp/nonexistent-grubenv-file")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
s.handleMetrics(w, req)
|
||||
|
||||
resp := w.Result()
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
output := string(body)
|
||||
|
||||
// Should still render with defaults
|
||||
if !strings.Contains(output, "kubesolo_os_boot_success 0") {
|
||||
t.Errorf("expected boot_success=0 with missing grubenv, got:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(output, "kubesolo_os_boot_counter 0") {
|
||||
t.Errorf("expected boot_counter=0 with missing grubenv, got:\n%s", output)
|
||||
}
|
||||
// active_slot should be empty
|
||||
if !strings.Contains(output, `active_slot=""`) {
|
||||
t.Errorf("expected empty active_slot with missing grubenv, got:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleMetricsUpdateNotAvailable(t *testing.T) {
|
||||
s := NewServer(":9100", "/tmp/nonexistent")
|
||||
// Don't call SetUpdateAvailable — should default to 0
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
s.handleMetrics(w, req)
|
||||
|
||||
resp := w.Result()
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
output := string(body)
|
||||
|
||||
if !strings.Contains(output, "kubesolo_os_update_available 0") {
|
||||
t.Errorf("expected update_available=0 by default, got:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(output, "kubesolo_os_update_last_check_timestamp_seconds 0") {
|
||||
t.Errorf("expected last_check=0 by default, got:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadGrubenvVar(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
grubenv := filepath.Join(dir, "grubenv")
|
||||
content := "active_slot=B\nboot_success=0\nboot_counter=2\nsome_other=value\n"
|
||||
if err := os.WriteFile(grubenv, []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := NewServer(":9100", grubenv)
|
||||
|
||||
tests := []struct {
|
||||
key string
|
||||
want string
|
||||
}{
|
||||
{"active_slot", "B"},
|
||||
{"boot_success", "0"},
|
||||
{"boot_counter", "2"},
|
||||
{"some_other", "value"},
|
||||
{"nonexistent", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := s.readGrubenvVar(tt.key)
|
||||
if got != tt.want {
|
||||
t.Errorf("readGrubenvVar(%q) = %q, want %q", tt.key, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadGrubenvVarMissingFile(t *testing.T) {
|
||||
s := NewServer(":9100", "/tmp/nonexistent-grubenv")
|
||||
got := s.readGrubenvVar("active_slot")
|
||||
if got != "" {
|
||||
t.Errorf("readGrubenvVar with missing file = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeInt(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
def string
|
||||
want string
|
||||
}{
|
||||
{"42", "0", "42"},
|
||||
{"0", "0", "0"},
|
||||
{"3", "0", "3"},
|
||||
{"", "0", "0"},
|
||||
{"abc", "0", "0"},
|
||||
{"1.5", "0", "0"},
|
||||
{"-1", "0", "-1"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := safeInt(tt.input, tt.def)
|
||||
if got != tt.want {
|
||||
t.Errorf("safeInt(%q, %q) = %q, want %q", tt.input, tt.def, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFileString(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// Test existing file
|
||||
path := filepath.Join(dir, "version")
|
||||
if err := os.WriteFile(path, []byte(" 1.2.3\n "), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readFileString(path)
|
||||
if got != "1.2.3" {
|
||||
t.Errorf("readFileString = %q, want %q", got, "1.2.3")
|
||||
}
|
||||
|
||||
// Test missing file
|
||||
got = readFileString("/tmp/nonexistent-file-12345")
|
||||
if got != "unknown" {
|
||||
t.Errorf("readFileString missing file = %q, want %q", got, "unknown")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user