feat: add production hardening — Ed25519 signing, Portainer Edge, SSH extension (Phase 4)

Image signing:
- Ed25519 sign/verify package (pure Go stdlib, zero deps)
- genkey and sign CLI subcommands for build system
- Optional --pubkey flag for verifying updates on apply
- Signature URLs in update metadata (latest.json)

Portainer Edge Agent:
- cloud-init portainer.go module writes K8s manifest
- Auto-deploys Edge Agent when portainer.edge-agent.enabled
- Full RBAC (ServiceAccount, ClusterRoleBinding, Deployment)
- 5 Portainer tests in portainer_test.go

Production tooling:
- SSH debug extension builder (hack/build-ssh-extension.sh)
- Boot performance benchmark (test/benchmark/bench-boot.sh)
- Resource usage benchmark (test/benchmark/bench-resources.sh)
- Deployment guide (docs/deployment-guide.md)

Test results: 50 update agent tests + 22 cloud-init tests passing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-11 11:26:23 -06:00
parent 8d25e1890e
commit 49a37e30e8
15 changed files with 1965 additions and 11 deletions

View File

@@ -33,6 +33,12 @@ func Apply(args []string) error {
client := image.NewClient(opts.ServerURL, stageDir)
defer client.Cleanup()
// Enable signature verification if public key is configured
if opts.PubKeyPath != "" {
client.SetPublicKeyPath(opts.PubKeyPath)
slog.Info("signature verification enabled", "pubkey", opts.PubKeyPath)
}
meta, err := client.CheckForUpdate()
if err != nil {
return fmt.Errorf("checking for update: %w", err)

View File

@@ -5,6 +5,7 @@ type opts struct {
ServerURL string
GrubenvPath string
TimeoutSecs int
PubKeyPath string
}
// parseOpts extracts command-line flags from args.
@@ -40,6 +41,11 @@ func parseOpts(args []string) opts {
}
i++
}
case "--pubkey":
if i+1 < len(args) {
o.PubKeyPath = args[i+1]
i++
}
}
}

75
update/cmd/sign.go Normal file
View File

@@ -0,0 +1,75 @@
package cmd
import (
"fmt"
"github.com/portainer/kubesolo-os/update/pkg/signing"
)
// Sign creates Ed25519 signatures for update artifacts.
// Used during the build process, not on the target device.
//
// Usage:
//
// kubesolo-update sign --key <privkey.hex> <file> [file...]
func Sign(args []string) error {
var keyPath string
var files []string
for i := 0; i < len(args); i++ {
switch args[i] {
case "--key":
if i+1 < len(args) {
keyPath = args[i+1]
i++
}
default:
// Non-flag args are files to sign
if args[i] != "" && args[i][0] != '-' {
files = append(files, args[i])
}
}
}
if keyPath == "" {
return fmt.Errorf("--key is required (path to Ed25519 private key hex file)")
}
if len(files) == 0 {
return fmt.Errorf("at least one file to sign is required")
}
signer, err := signing.NewSignerFromFile(keyPath)
if err != nil {
return fmt.Errorf("loading private key: %w", err)
}
for _, f := range files {
sigPath := f + ".sig"
if err := signer.SignFile(f, sigPath); err != nil {
return fmt.Errorf("signing %s: %w", f, err)
}
fmt.Printf("Signed: %s → %s\n", f, sigPath)
}
return nil
}
// GenKey generates a new Ed25519 key pair for signing updates.
//
// Usage:
//
// kubesolo-update genkey
func GenKey(args []string) error {
pub, priv, err := signing.GenerateKeyPair()
if err != nil {
return err
}
fmt.Printf("Public key (hex): %s\n", pub)
fmt.Printf("Private key (hex): %s\n", priv)
fmt.Println()
fmt.Println("Save the public key to /etc/kubesolo/update-pubkey.hex on the device.")
fmt.Println("Keep the private key secure and offline — use it only for signing updates.")
return nil
}

View File

@@ -8,6 +8,8 @@
// kubesolo-update rollback Force rollback to other partition
// kubesolo-update healthcheck Post-boot health verification
// kubesolo-update status Show current A/B slot and boot status
// kubesolo-update sign Sign update artifacts with Ed25519 key
// kubesolo-update genkey Generate new Ed25519 signing key pair
package main
import (
@@ -42,6 +44,10 @@ func main() {
err = cmd.Healthcheck(os.Args[2:])
case "status":
err = cmd.Status(os.Args[2:])
case "sign":
err = cmd.Sign(os.Args[2:])
case "genkey":
err = cmd.GenKey(os.Args[2:])
default:
fmt.Fprintf(os.Stderr, "unknown command: %s\n\n", os.Args[1])
usage()
@@ -64,15 +70,18 @@ Commands:
rollback Force rollback to other partition
healthcheck Post-boot health verification (marks boot successful)
status Show current A/B slot and boot status
sign Sign artifacts with Ed25519 private key (build system)
genkey Generate new Ed25519 signing key pair
Options:
--server URL Update server URL (default: from /etc/kubesolo/update.conf)
--grubenv PATH Path to grubenv file (default: /boot/grub/grubenv)
--timeout SECS Health check timeout in seconds (default: 120)
--pubkey PATH Ed25519 public key for signature verification (optional)
Examples:
kubesolo-update check --server https://updates.example.com
kubesolo-update apply --server https://updates.example.com
kubesolo-update apply --server https://updates.example.com --pubkey /etc/kubesolo/update-pubkey.hex
kubesolo-update healthcheck
kubesolo-update status
`)

View File

@@ -19,17 +19,22 @@ import (
"os"
"path/filepath"
"time"
"github.com/portainer/kubesolo-os/update/pkg/signing"
)
// UpdateMetadata describes an available update from the update server.
type UpdateMetadata struct {
Version string `json:"version"`
VmlinuzURL string `json:"vmlinuz_url"`
VmlinuzSHA256 string `json:"vmlinuz_sha256"`
InitramfsURL string `json:"initramfs_url"`
Version string `json:"version"`
VmlinuzURL string `json:"vmlinuz_url"`
VmlinuzSHA256 string `json:"vmlinuz_sha256"`
VmlinuzSigURL string `json:"vmlinuz_sig_url,omitempty"`
InitramfsURL string `json:"initramfs_url"`
InitramfsSHA256 string `json:"initramfs_sha256"`
ReleaseNotes string `json:"release_notes,omitempty"`
ReleaseDate string `json:"release_date,omitempty"`
InitramfsSigURL string `json:"initramfs_sig_url,omitempty"`
MetadataSigURL string `json:"metadata_sig_url,omitempty"`
ReleaseNotes string `json:"release_notes,omitempty"`
ReleaseDate string `json:"release_date,omitempty"`
}
// StagedImage represents downloaded and verified update files.
@@ -41,9 +46,10 @@ type StagedImage struct {
// Client handles communication with the update server.
type Client struct {
serverURL string
httpClient *http.Client
stageDir string
serverURL string
httpClient *http.Client
stageDir string
pubKeyPath string // path to Ed25519 public key for signature verification
}
// NewClient creates a new update image client.
@@ -57,6 +63,13 @@ func NewClient(serverURL, stageDir string) *Client {
}
}
// SetPublicKeyPath sets the path to the Ed25519 public key used
// for verifying update signatures. If set, downloaded images will
// be verified against their .sig files from the update server.
func (c *Client) SetPublicKeyPath(path string) {
c.pubKeyPath = path
}
// CheckForUpdate fetches the latest update metadata from the server.
func (c *Client) CheckForUpdate() (*UpdateMetadata, error) {
url := c.serverURL + "/latest.json"
@@ -103,6 +116,15 @@ func (c *Client) Download(meta *UpdateMetadata) (*StagedImage, error) {
return nil, fmt.Errorf("downloading initramfs: %w", err)
}
// Verify signatures if public key is configured
if c.pubKeyPath != "" {
if err := c.verifySignatures(meta, vmlinuzPath, initramfsPath); err != nil {
os.Remove(vmlinuzPath)
os.Remove(initramfsPath)
return nil, fmt.Errorf("signature verification: %w", err)
}
}
return &StagedImage{
VmlinuzPath: vmlinuzPath,
InitramfsPath: initramfsPath,
@@ -159,6 +181,60 @@ func (c *Client) downloadAndVerify(url, dest, expectedSHA256 string) error {
return nil
}
// verifySignatures downloads .sig files and verifies them against the staged images.
func (c *Client) verifySignatures(meta *UpdateMetadata, vmlinuzPath, initramfsPath string) error {
verifier, err := signing.NewVerifierFromFile(c.pubKeyPath)
if err != nil {
return fmt.Errorf("loading public key: %w", err)
}
// Verify vmlinuz signature
if meta.VmlinuzSigURL != "" {
sigPath := vmlinuzPath + ".sig"
if err := c.downloadToFile(meta.VmlinuzSigURL, sigPath); err != nil {
return fmt.Errorf("downloading vmlinuz signature: %w", err)
}
if err := verifier.VerifyFile(vmlinuzPath, sigPath); err != nil {
return fmt.Errorf("vmlinuz: %w", err)
}
slog.Info("vmlinuz signature verified")
}
// Verify initramfs signature
if meta.InitramfsSigURL != "" {
sigPath := initramfsPath + ".sig"
if err := c.downloadToFile(meta.InitramfsSigURL, sigPath); err != nil {
return fmt.Errorf("downloading initramfs signature: %w", err)
}
if err := verifier.VerifyFile(initramfsPath, sigPath); err != nil {
return fmt.Errorf("initramfs: %w", err)
}
slog.Info("initramfs signature verified")
}
return nil
}
// downloadToFile downloads a URL to a local file (used for signature files).
func (c *Client) downloadToFile(url, dest string) error {
resp, err := c.httpClient.Get(url)
if err != nil {
return fmt.Errorf("downloading %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("server returned %d for %s", resp.StatusCode, url)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("reading response: %w", err)
}
return os.WriteFile(dest, data, 0o644)
}
// VerifyFile checks the SHA256 checksum of an existing file.
func VerifyFile(path, expectedSHA256 string) error {
f, err := os.Open(path)

View File

@@ -0,0 +1,188 @@
// Package signing provides Ed25519 signature verification for update images.
//
// KubeSolo OS uses Ed25519 signatures to ensure update integrity and
// authenticity. The update server signs both the metadata (latest.json)
// and individual image files. The update agent verifies signatures using
// a trusted public key embedded at build time or loaded from disk.
//
// Signature workflow:
// 1. Build system signs images with private key (offline)
// 2. Signatures stored alongside images on update server (.sig files)
// 3. Update agent downloads signatures and verifies before applying
//
// Key format: raw 32-byte Ed25519 public keys, hex-encoded for config files.
package signing
import (
"crypto/ed25519"
"encoding/hex"
"fmt"
"log/slog"
"os"
)
// Verifier checks Ed25519 signatures on update artifacts.
type Verifier struct {
publicKey ed25519.PublicKey
}
// NewVerifier creates a verifier from a hex-encoded Ed25519 public key string.
func NewVerifier(hexPubKey string) (*Verifier, error) {
keyBytes, err := hex.DecodeString(hexPubKey)
if err != nil {
return nil, fmt.Errorf("decoding public key hex: %w", err)
}
if len(keyBytes) != ed25519.PublicKeySize {
return nil, fmt.Errorf("invalid public key size: got %d bytes, want %d", len(keyBytes), ed25519.PublicKeySize)
}
return &Verifier{publicKey: ed25519.PublicKey(keyBytes)}, nil
}
// NewVerifierFromFile reads an Ed25519 public key from a file.
// The file should contain the hex-encoded public key (64 hex chars).
func NewVerifierFromFile(path string) (*Verifier, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading public key file: %w", err)
}
// Trim whitespace
hexKey := trimWhitespace(string(data))
return NewVerifier(hexKey)
}
// VerifyFile checks that a file's signature matches.
// The signature file should contain the raw Ed25519 signature (64 bytes)
// or a hex-encoded signature (128 hex chars).
func (v *Verifier) VerifyFile(filePath, sigPath string) error {
// Read the file content
message, err := os.ReadFile(filePath)
if err != nil {
return fmt.Errorf("reading file: %w", err)
}
// Read the signature
sigData, err := os.ReadFile(sigPath)
if err != nil {
return fmt.Errorf("reading signature: %w", err)
}
sig, err := decodeSignature(sigData)
if err != nil {
return fmt.Errorf("decoding signature: %w", err)
}
if !ed25519.Verify(v.publicKey, message, sig) {
return fmt.Errorf("signature verification failed for %s", filePath)
}
slog.Debug("signature verified", "file", filePath)
return nil
}
// VerifyBytes checks a signature against raw bytes.
func (v *Verifier) VerifyBytes(message, signature []byte) error {
if !ed25519.Verify(v.publicKey, message, signature) {
return fmt.Errorf("signature verification failed")
}
return nil
}
// Signer creates Ed25519 signatures for update artifacts.
// Used by the build system, not the update agent on the device.
type Signer struct {
privateKey ed25519.PrivateKey
}
// NewSigner creates a signer from a hex-encoded Ed25519 private key.
func NewSigner(hexPrivKey string) (*Signer, error) {
keyBytes, err := hex.DecodeString(hexPrivKey)
if err != nil {
return nil, fmt.Errorf("decoding private key hex: %w", err)
}
if len(keyBytes) != ed25519.PrivateKeySize {
return nil, fmt.Errorf("invalid private key size: got %d bytes, want %d", len(keyBytes), ed25519.PrivateKeySize)
}
return &Signer{privateKey: ed25519.PrivateKey(keyBytes)}, nil
}
// NewSignerFromFile reads an Ed25519 private key from a file.
func NewSignerFromFile(path string) (*Signer, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading private key file: %w", err)
}
hexKey := trimWhitespace(string(data))
return NewSigner(hexKey)
}
// SignFile creates a signature for a file and writes it to sigPath.
func (s *Signer) SignFile(filePath, sigPath string) error {
message, err := os.ReadFile(filePath)
if err != nil {
return fmt.Errorf("reading file: %w", err)
}
sig := ed25519.Sign(s.privateKey, message)
// Write hex-encoded signature
hexSig := hex.EncodeToString(sig)
if err := os.WriteFile(sigPath, []byte(hexSig+"\n"), 0o644); err != nil {
return fmt.Errorf("writing signature: %w", err)
}
slog.Debug("signed", "file", filePath, "sig", sigPath)
return nil
}
// SignBytes creates a signature for raw bytes.
func (s *Signer) SignBytes(message []byte) []byte {
return ed25519.Sign(s.privateKey, message)
}
// PublicKeyHex returns the hex-encoded public key corresponding to this signer.
func (s *Signer) PublicKeyHex() string {
pubKey := s.privateKey.Public().(ed25519.PublicKey)
return hex.EncodeToString(pubKey)
}
// GenerateKeyPair creates a new Ed25519 key pair and returns hex-encoded strings.
func GenerateKeyPair() (publicKeyHex, privateKeyHex string, err error) {
pub, priv, err := ed25519.GenerateKey(nil)
if err != nil {
return "", "", fmt.Errorf("generating key pair: %w", err)
}
return hex.EncodeToString(pub), hex.EncodeToString(priv), nil
}
// decodeSignature handles both raw (64 bytes) and hex-encoded signatures.
func decodeSignature(data []byte) ([]byte, error) {
// Trim whitespace for hex-encoded sigs
trimmed := trimWhitespace(string(data))
// If exactly 64 bytes, treat as raw signature
if len(data) == ed25519.SignatureSize {
return data, nil
}
// Try hex decode
sig, err := hex.DecodeString(trimmed)
if err != nil {
return nil, fmt.Errorf("invalid signature format (not raw or hex): %w", err)
}
if len(sig) != ed25519.SignatureSize {
return nil, fmt.Errorf("invalid signature size: got %d bytes, want %d", len(sig), ed25519.SignatureSize)
}
return sig, nil
}
func trimWhitespace(s string) string {
result := make([]byte, 0, len(s))
for _, b := range []byte(s) {
if b != ' ' && b != '\n' && b != '\r' && b != '\t' {
result = append(result, b)
}
}
return string(result)
}

View File

@@ -0,0 +1,334 @@
package signing
import (
"crypto/ed25519"
"encoding/hex"
"os"
"path/filepath"
"testing"
)
func generateTestKeyPair(t *testing.T) (string, string) {
t.Helper()
pub, priv, err := GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
return pub, priv
}
func TestGenerateKeyPair(t *testing.T) {
pub, priv, err := GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
// Public key should be 32 bytes = 64 hex chars
if len(pub) != 64 {
t.Errorf("expected 64 hex chars for public key, got %d", len(pub))
}
// Private key should be 64 bytes = 128 hex chars
if len(priv) != 128 {
t.Errorf("expected 128 hex chars for private key, got %d", len(priv))
}
// Keys should be valid hex
if _, err := hex.DecodeString(pub); err != nil {
t.Errorf("public key is not valid hex: %v", err)
}
if _, err := hex.DecodeString(priv); err != nil {
t.Errorf("private key is not valid hex: %v", err)
}
}
func TestNewVerifier(t *testing.T) {
pub, _ := generateTestKeyPair(t)
v, err := NewVerifier(pub)
if err != nil {
t.Fatal(err)
}
if v == nil {
t.Fatal("verifier should not be nil")
}
}
func TestNewVerifierInvalid(t *testing.T) {
// Invalid hex
_, err := NewVerifier("not-hex")
if err == nil {
t.Fatal("expected error for invalid hex")
}
// Wrong length
_, err = NewVerifier("abcd")
if err == nil {
t.Fatal("expected error for wrong key length")
}
}
func TestNewSigner(t *testing.T) {
_, priv := generateTestKeyPair(t)
s, err := NewSigner(priv)
if err != nil {
t.Fatal(err)
}
if s == nil {
t.Fatal("signer should not be nil")
}
}
func TestSignAndVerifyBytes(t *testing.T) {
pub, priv := generateTestKeyPair(t)
signer, err := NewSigner(priv)
if err != nil {
t.Fatal(err)
}
verifier, err := NewVerifier(pub)
if err != nil {
t.Fatal(err)
}
message := []byte("KubeSolo OS update v1.0.0")
sig := signer.SignBytes(message)
// Verify should succeed
if err := verifier.VerifyBytes(message, sig); err != nil {
t.Errorf("verification should succeed: %v", err)
}
// Tampered message should fail
tampered := []byte("KubeSolo OS update v1.0.1")
if err := verifier.VerifyBytes(tampered, sig); err == nil {
t.Error("verification should fail for tampered message")
}
// Tampered signature should fail
badSig := make([]byte, len(sig))
copy(badSig, sig)
badSig[0] ^= 0xff
if err := verifier.VerifyBytes(message, badSig); err == nil {
t.Error("verification should fail for tampered signature")
}
}
func TestSignAndVerifyFile(t *testing.T) {
pub, priv := generateTestKeyPair(t)
dir := t.TempDir()
signer, err := NewSigner(priv)
if err != nil {
t.Fatal(err)
}
verifier, err := NewVerifier(pub)
if err != nil {
t.Fatal(err)
}
// Create a test file
filePath := filepath.Join(dir, "test-image.gz")
content := []byte("fake OS image content for signing test")
if err := os.WriteFile(filePath, content, 0o644); err != nil {
t.Fatal(err)
}
// Sign
sigPath := filePath + ".sig"
if err := signer.SignFile(filePath, sigPath); err != nil {
t.Fatal(err)
}
// Verify signature file was created
if _, err := os.Stat(sigPath); err != nil {
t.Fatalf("signature file not created: %v", err)
}
// Verify
if err := verifier.VerifyFile(filePath, sigPath); err != nil {
t.Errorf("verification should succeed: %v", err)
}
}
func TestVerifyFileTampered(t *testing.T) {
pub, priv := generateTestKeyPair(t)
dir := t.TempDir()
signer, err := NewSigner(priv)
if err != nil {
t.Fatal(err)
}
verifier, err := NewVerifier(pub)
if err != nil {
t.Fatal(err)
}
// Create and sign a file
filePath := filepath.Join(dir, "test-image.gz")
if err := os.WriteFile(filePath, []byte("original content"), 0o644); err != nil {
t.Fatal(err)
}
sigPath := filePath + ".sig"
if err := signer.SignFile(filePath, sigPath); err != nil {
t.Fatal(err)
}
// Tamper with the file
if err := os.WriteFile(filePath, []byte("tampered content"), 0o644); err != nil {
t.Fatal(err)
}
// Verification should fail
if err := verifier.VerifyFile(filePath, sigPath); err == nil {
t.Error("verification should fail for tampered file")
}
}
func TestVerifyFileWrongKey(t *testing.T) {
_, priv := generateTestKeyPair(t)
otherPub, _ := generateTestKeyPair(t) // different key pair
dir := t.TempDir()
signer, err := NewSigner(priv)
if err != nil {
t.Fatal(err)
}
wrongVerifier, err := NewVerifier(otherPub)
if err != nil {
t.Fatal(err)
}
// Create and sign
filePath := filepath.Join(dir, "test.gz")
if err := os.WriteFile(filePath, []byte("test content"), 0o644); err != nil {
t.Fatal(err)
}
sigPath := filePath + ".sig"
if err := signer.SignFile(filePath, sigPath); err != nil {
t.Fatal(err)
}
// Verify with wrong key should fail
if err := wrongVerifier.VerifyFile(filePath, sigPath); err == nil {
t.Error("verification should fail with wrong public key")
}
}
func TestNewVerifierFromFile(t *testing.T) {
pub, _ := generateTestKeyPair(t)
dir := t.TempDir()
keyFile := filepath.Join(dir, "pubkey.hex")
if err := os.WriteFile(keyFile, []byte(pub+"\n"), 0o644); err != nil {
t.Fatal(err)
}
v, err := NewVerifierFromFile(keyFile)
if err != nil {
t.Fatal(err)
}
if v == nil {
t.Fatal("verifier should not be nil")
}
}
func TestNewSignerFromFile(t *testing.T) {
_, priv := generateTestKeyPair(t)
dir := t.TempDir()
keyFile := filepath.Join(dir, "privkey.hex")
if err := os.WriteFile(keyFile, []byte(priv+"\n"), 0o644); err != nil {
t.Fatal(err)
}
s, err := NewSignerFromFile(keyFile)
if err != nil {
t.Fatal(err)
}
if s == nil {
t.Fatal("signer should not be nil")
}
}
func TestSignerPublicKeyHex(t *testing.T) {
pub, priv := generateTestKeyPair(t)
signer, err := NewSigner(priv)
if err != nil {
t.Fatal(err)
}
got := signer.PublicKeyHex()
if got != pub {
t.Errorf("public key mismatch: got %s, want %s", got, pub)
}
}
func TestDecodeSignature(t *testing.T) {
// Create a valid signature
_, priv, _ := ed25519.GenerateKey(nil)
message := []byte("test")
rawSig := ed25519.Sign(priv, message)
hexSig := hex.EncodeToString(rawSig)
// Raw signature (64 bytes)
decoded, err := decodeSignature(rawSig)
if err != nil {
t.Fatalf("raw sig decode failed: %v", err)
}
if len(decoded) != ed25519.SignatureSize {
t.Errorf("expected %d bytes, got %d", ed25519.SignatureSize, len(decoded))
}
// Hex-encoded signature
decoded, err = decodeSignature([]byte(hexSig))
if err != nil {
t.Fatalf("hex sig decode failed: %v", err)
}
if len(decoded) != ed25519.SignatureSize {
t.Errorf("expected %d bytes, got %d", ed25519.SignatureSize, len(decoded))
}
// Hex with trailing newline
decoded, err = decodeSignature([]byte(hexSig + "\n"))
if err != nil {
t.Fatalf("hex sig with newline decode failed: %v", err)
}
if len(decoded) != ed25519.SignatureSize {
t.Errorf("expected %d bytes, got %d", ed25519.SignatureSize, len(decoded))
}
// Invalid data
_, err = decodeSignature([]byte("not valid"))
if err == nil {
t.Error("expected error for invalid signature data")
}
}
func TestTrimWhitespace(t *testing.T) {
tests := []struct {
input string
expect string
}{
{"hello", "hello"},
{" hello ", "hello"},
{"hello\n", "hello"},
{"\thello\r\n", "hello"},
{" he llo ", "hello"},
}
for _, tt := range tests {
got := trimWhitespace(tt.input)
if got != tt.expect {
t.Errorf("trimWhitespace(%q) = %q, want %q", tt.input, got, tt.expect)
}
}
}