adelorenzo/kilo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pkg/{k8s,mesh}: introduce liveness checks

Browse Source 
This commit introduces liveness checks to Kilo. This allows the Kilo
daemons to take nodes with inactive or dead Kilo deamons out of the
topology until they are alive again.
This commit is contained in:
Lucas Servén Marín
2019-04-02 18:25:08 +02:00
parent a8467f779e
commit 72bfb762b9
4 changed files with 64 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/k8s/backend.go
View File

@@ -20,10 +20,11 @@ import (
"fmt"
"net"
"path"
"strconv"
"strings"
"time"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch"
@@ -42,6 +43,7 @@ const (
forceExternalIPAnnotationKey = "kilo.squat.ai/force-external-ip"
internalIPAnnotationKey = "kilo.squat.ai/internal-ip"
keyAnnotationKey = "kilo.squat.ai/key"
lastSeenAnnotationKey = "kilo.squat.ai/last-seen"
leaderAnnotationKey = "kilo.squat.ai/leader"
locationAnnotationKey = "kilo.squat.ai/location"
regionLabelKey = "failure-domain.beta.kubernetes.io/region"
@@ -76,6 +78,7 @@ func (b *backend) CleanUp(name string) error {
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(externalIPAnnotationKey, "/", jsonPatchSlash, 1))),
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(internalIPAnnotationKey, "/", jsonPatchSlash, 1))),
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(keyAnnotationKey, "/", jsonPatchSlash, 1))),
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(lastSeenAnnotationKey, "/", jsonPatchSlash, 1))),
}, ",") + "]")
if _, err := b.client.CoreV1().Nodes().Patch(name, types.JSONPatchType, patch); err != nil {
return fmt.Errorf("failed to patch node: %v", err)
@@ -155,6 +158,7 @@ func (b *backend) Set(name string, node *mesh.Node) error {
n.ObjectMeta.Annotations[externalIPAnnotationKey] = node.ExternalIP.String()
n.ObjectMeta.Annotations[internalIPAnnotationKey] = node.InternalIP.String()
n.ObjectMeta.Annotations[keyAnnotationKey] = string(node.Key)
n.ObjectMeta.Annotations[lastSeenAnnotationKey] = strconv.FormatInt(node.LastSeen, 10)
oldData, err := json.Marshal(old)
if err != nil {
return err
@@ -200,6 +204,14 @@ func translateNode(node *v1.Node) *mesh.Node {
if !ok {
externalIP = node.ObjectMeta.Annotations[externalIPAnnotationKey]
}
var lastSeen int64
if ls, ok := node.ObjectMeta.Annotations[lastSeenAnnotationKey]; !ok {
lastSeen = 0
} else {
if lastSeen, err = strconv.ParseInt(ls, 10, 64); err != nil {
lastSeen = 0
}
}
return &mesh.Node{
// ExternalIP and InternalIP should only ever fail to parse if the
// remote node's mesh has not yet set its IP address;
@@ -208,6 +220,7 @@ func translateNode(node *v1.Node) *mesh.Node {
ExternalIP: normalizeIP(externalIP),
InternalIP: normalizeIP(node.ObjectMeta.Annotations[internalIPAnnotationKey]),
Key: []byte(node.ObjectMeta.Annotations[keyAnnotationKey]),
LastSeen: lastSeen,
Leader: leader,
Location: location,
Name: node.Name,

17
pkg/k8s/backend_test.go
View File

@@ -19,7 +19,7 @@ import (
"testing"
"github.com/kylelemons/godebug/pretty"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"github.com/squat/kilo/pkg/mesh"
)
@@ -38,15 +38,15 @@ func TestTranslateNode(t *testing.T) {
out: &mesh.Node{},
},
{
name: "invalid ip",
name: "invalid ips",
annotations: map[string]string{
externalIPAnnotationKey: "10.0.0.1",
internalIPAnnotationKey: "10.0.0.1",
internalIPAnnotationKey: "foo",
},
out: &mesh.Node{},
},
{
name: "valid ip",
name: "valid ips",
annotations: map[string]string{
externalIPAnnotationKey: "10.0.0.1/24",
internalIPAnnotationKey: "10.0.0.2/32",
@@ -109,6 +109,13 @@ func TestTranslateNode(t *testing.T) {
ExternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
},
},
{
name: "invalid time",
annotations: map[string]string{
lastSeenAnnotationKey: "foo",
},
out: &mesh.Node{},
},
{
name: "complete",
annotations: map[string]string{
@@ -116,6 +123,7 @@ func TestTranslateNode(t *testing.T) {
forceExternalIPAnnotationKey: "10.0.0.2/24",
internalIPAnnotationKey: "10.0.0.2/32",
keyAnnotationKey: "foo",
lastSeenAnnotationKey: "1000000000",
leaderAnnotationKey: "",
locationAnnotationKey: "b",
},
@@ -126,6 +134,7 @@ func TestTranslateNode(t *testing.T) {
ExternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
Key: []byte("foo"),
LastSeen: 1000000000,
Leader: true,
Location: "b",
Subnet: &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},