adelorenzo
/
kilo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pkg/mesh,cmd: add WireGuard IP to Nodes 

This allows admins or users to have some easy visibility into the
configuration of the Kilo cluster.
This commit is contained in:
Lucas Servén Marín 2019-05-10 02:05:57 +02:00
parent b04264ecc1
commit 4d9c203603
No known key found for this signature in database
GPG Key ID: 586FEAF680DA74AD
8 changed files with 84 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kg/main.go
View File

@ -87,7 +87,7 @@ func Main() error {
master := flag.String("master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
var port uint
flag.UintVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
subnet := flag.String("subnet", "10.4.0.0/16", "CIDR from which to allocate addresses for WireGuard interfaces.")
subnet := flag.String("subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.")
printVersion := flag.Bool("version", false, "Print version and exit")
flag.Parse()

7
cmd/kgctl/graph.go
View File

@ -35,17 +35,22 @@ func runGraph(_ *cobra.Command, _ []string) error {
return fmt.Errorf("failed to list nodes: %v", err)
}
var hostname string
subnet := mesh.DefaultKiloSubnet
nodes := make(map[string]*mesh.Node)
for _, n := range ns {
if n.Ready() {
nodes[n.Name] = n
hostname = n.Name
}
if n.WireGuardIP != nil {
subnet = n.WireGuardIP
}
}
subnet.IP = subnet.IP.Mask(subnet.Mask)
if len(nodes) == 0 {
return fmt.Errorf("did not find any valid Kilo nodes in the cluster")
}
t, err := mesh.NewTopology(nodes, nil, opts.granularity, hostname, 0, []byte{}, opts.subnet)
t, err := mesh.NewTopology(nodes, nil, opts.granularity, hostname, 0, []byte{}, subnet)
if err != nil {
return fmt.Errorf("failed to create topology: %v", err)
}

10
cmd/kgctl/main.go
View File

@ -16,7 +16,6 @@ package main
import (
"fmt"
"net"
"os"
"strings"
@ -59,21 +58,13 @@ var (
opts struct {
backend mesh.Backend
granularity mesh.Granularity
subnet *net.IPNet
}
backend string
granularity string
kubeconfig string
subnet string
)
func runRoot(_ *cobra.Command, _ []string) error {
_, s, err := net.ParseCIDR(subnet)
if err != nil {
return fmt.Errorf("failed to parse %q as CIDR: %v", subnet, err)
}
opts.subnet = s
opts.granularity = mesh.Granularity(granularity)
switch opts.granularity {
case mesh.LogicalGranularity:
@ -117,7 +108,6 @@ func main() {
cmd.PersistentFlags().StringVar(&backend, "backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig.")
cmd.PersistentFlags().StringVar(&subnet, "subnet", "10.4.0.0/16", "CIDR from which to allocate addressees to WireGuard interfaces.")
for _, subCmd := range []*cobra.Command{
graph(),

14
cmd/kgctl/showconf.go
View File

@ -121,12 +121,17 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
return fmt.Errorf("failed to list peers: %v", err)
}
hostname := args[0]
subnet := mesh.DefaultKiloSubnet
nodes := make(map[string]*mesh.Node)
for _, n := range ns {
if n.Ready() {
nodes[n.Name] = n
}
if n.WireGuardIP != nil {
subnet = n.WireGuardIP
}
}
subnet.IP = subnet.IP.Mask(subnet.Mask)
if len(nodes) == 0 {
return errors.New("did not find any valid Kilo nodes in the cluster")
}
@ -141,7 +146,7 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
}
}
t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, opts.subnet)
t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet)
if err != nil {
return fmt.Errorf("failed to create topology: %v", err)
}
@ -192,13 +197,18 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
return fmt.Errorf("failed to list peers: %v", err)
}
var hostname string
subnet := mesh.DefaultKiloSubnet
nodes := make(map[string]*mesh.Node)
for _, n := range ns {
if n.Ready() {
nodes[n.Name] = n
hostname = n.Name
}
if n.WireGuardIP != nil {
subnet = n.WireGuardIP
}
}
subnet.IP = subnet.IP.Mask(subnet.Mask)
if len(nodes) == 0 {
return errors.New("did not find any valid Kilo nodes in the cluster")
}
@ -214,7 +224,7 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
return fmt.Errorf("did not find any peer named %q in the cluster", peer)
}
t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, opts.subnet)
t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet)
if err != nil {
return fmt.Errorf("failed to create topology: %v", err)
}

18
pkg/k8s/backend.go
View File

@ -55,9 +55,11 @@ const (
lastSeenAnnotationKey = "kilo.squat.ai/last-seen"
leaderAnnotationKey = "kilo.squat.ai/leader"
locationAnnotationKey = "kilo.squat.ai/location"
regionLabelKey = "failure-domain.beta.kubernetes.io/region"
jsonPatchSlash = "~1"
jsonRemovePatch = `{"op": "remove", "path": "%s"}`
wireGuardIPAnnotationKey = "kilo.squat.ai/wireguard-ip"
regionLabelKey = "failure-domain.beta.kubernetes.io/region"
jsonPatchSlash = "~1"
jsonRemovePatch = `{"op": "remove", "path": "%s"}`
)
type backend struct {
@ -119,6 +121,7 @@ func (nb *nodeBackend) CleanUp(name string) error {
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(internalIPAnnotationKey, "/", jsonPatchSlash, 1))),
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(keyAnnotationKey, "/", jsonPatchSlash, 1))),
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(lastSeenAnnotationKey, "/", jsonPatchSlash, 1))),
fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(wireGuardIPAnnotationKey, "/", jsonPatchSlash, 1))),
}, ",") + "]")
if _, err := nb.client.CoreV1().Nodes().Patch(name, types.JSONPatchType, patch); err != nil {
return fmt.Errorf("failed to patch node: %v", err)
@ -204,6 +207,11 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
n.ObjectMeta.Annotations[internalIPAnnotationKey] = node.InternalIP.String()
n.ObjectMeta.Annotations[keyAnnotationKey] = string(node.Key)
n.ObjectMeta.Annotations[lastSeenAnnotationKey] = strconv.FormatInt(node.LastSeen, 10)
if node.WireGuardIP == nil {
n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = ""
} else {
n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = node.WireGuardIP.String()
}
oldData, err := json.Marshal(old)
if err != nil {
return err
@ -270,6 +278,10 @@ func translateNode(node *v1.Node) *mesh.Node {
Location: location,
Name: node.Name,
Subnet: subnet,
// WireGuardIP can fail to parse if the node is not a leader or if
// the node's agent has not yet reconciled. In either case, the IP
// will parse as nil.
WireGuardIP: normalizeIP(node.ObjectMeta.Annotations[wireGuardIPAnnotationKey]),
}
}

16
pkg/k8s/backend_test.go
View File

@ -128,18 +128,20 @@ func TestTranslateNode(t *testing.T) {
lastSeenAnnotationKey: "1000000000",
leaderAnnotationKey: "",
locationAnnotationKey: "b",
wireGuardIPAnnotationKey: "10.4.0.1/16",
},
labels: map[string]string{
regionLabelKey: "a",
},
out: &mesh.Node{
ExternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
Key: []byte("foo"),
LastSeen: 1000000000,
Leader: true,
Location: "b",
Subnet: &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
ExternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
Key: []byte("foo"),
LastSeen: 1000000000,
Leader: true,
Location: "b",
Subnet: &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
WireGuardIP: &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
},
subnet: "10.2.1.0/24",
},

35
pkg/mesh/mesh.go
View File

@ -49,6 +49,9 @@ const (
DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist"
)
// DefaultKiloSubnet is the default CIDR for Kilo.
var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}}
// Granularity represents the abstraction level at which the network
// should be meshed.
type Granularity string
@ -86,14 +89,16 @@ type Node struct {
LastSeen int64
// Leader is a suggestion to Kilo that
// the node wants to lead its segment.
Leader bool
Location string
Name string
Subnet *net.IPNet
Leader bool
Location string
Name string
Subnet *net.IPNet
WireGuardIP *net.IPNet
}
// Ready indicates whether or not the node is ready.
func (n *Node) Ready() bool {
// Nodes that are not leaders will not have WireGuardIPs, so it is not required.
return n != nil && n.ExternalIP != nil && n.Key != nil && n.InternalIP != nil && n.Subnet != nil && time.Now().Unix()-n.LastSeen < int64(resyncPeriod)*2/int64(time.Second)
}
@ -194,6 +199,7 @@ type Mesh struct {
subnet *net.IPNet
table *route.Table
tunlIface int
wireGuardIP *net.IPNet
// nodes and peers are mutable fields in the struct
// and needs to be guarded.
@ -514,14 +520,15 @@ func (m *Mesh) handleLocal(n *Node) {
// Take leader, location, and subnet from the argument, as these
// are not determined by kilo.
local := &Node{
ExternalIP: n.ExternalIP,
Key: m.pub,
InternalIP: m.internalIP,
LastSeen: time.Now().Unix(),
Leader: n.Leader,
Location: n.Location,
Name: m.hostname,
Subnet: n.Subnet,
ExternalIP: n.ExternalIP,
Key: m.pub,
InternalIP: m.internalIP,
LastSeen: time.Now().Unix(),
Leader: n.Leader,
Location: n.Location,
Name: m.hostname,
Subnet: n.Subnet,
WireGuardIP: m.wireGuardIP,
}
if !nodesAreEqual(n, local) {
level.Debug(m.logger).Log("msg", "local node differs from backend")
@ -583,6 +590,8 @@ func (m *Mesh) applyTopology() {
m.errorCounter.WithLabelValues("apply").Inc()
return
}
// Update the node's WireGuard IP.
m.wireGuardIP = t.wireGuardCIDR
conf := t.Conf()
buf, err := conf.Bytes()
if err != nil {
@ -740,7 +749,7 @@ func nodesAreEqual(a, b *Node) bool {
// Ignore LastSeen when comparing equality we want to check if the nodes are
// equivalent. However, we do want to check if LastSeen has transitioned
// between valid and invalid.
return ipNetsEqual(a.ExternalIP, b.ExternalIP) && string(a.Key) == string(b.Key) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready()
return ipNetsEqual(a.ExternalIP, b.ExternalIP) && string(a.Key) == string(b.Key) && ipNetsEqual(a.WireGuardIP, b.WireGuardIP) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready()
}
func peersAreEqual(a, b *Peer) bool {

39
pkg/mesh/topology_test.go
View File

@ -29,9 +29,8 @@ func allowedIPs(ips ...string) string {
return strings.Join(ips, ", ")
}
func setup(t *testing.T) (map[string]*Node, map[string]*Peer, []byte, uint32, *net.IPNet) {
func setup(t *testing.T) (map[string]*Node, map[string]*Peer, []byte, uint32) {
key := []byte("private")
kiloNet := &net.IPNet{IP: net.ParseIP("10.4.0.0").To4(), Mask: net.CIDRMask(16, 32)}
e1 := &net.IPNet{IP: net.ParseIP("10.1.0.1").To4(), Mask: net.CIDRMask(16, 32)}
e2 := &net.IPNet{IP: net.ParseIP("10.1.0.2").To4(), Mask: net.CIDRMask(16, 32)}
e3 := &net.IPNet{IP: net.ParseIP("10.1.0.3").To4(), Mask: net.CIDRMask(16, 32)}
@ -89,11 +88,11 @@ func setup(t *testing.T) (map[string]*Node, map[string]*Peer, []byte, uint32, *n
},
},
}
return nodes, peers, key, DefaultKiloPort, kiloNet
return nodes, peers, key, DefaultKiloPort
}
func TestNewTopology(t *testing.T) {
nodes, peers, key, port, kiloNet := setup(t)
nodes, peers, key, port := setup(t)
w1 := net.ParseIP("10.4.0.1").To4()
w2 := net.ParseIP("10.4.0.2").To4()
@ -112,7 +111,7 @@ func TestNewTopology(t *testing.T) {
hostname: nodes["a"].Name,
leader: true,
location: nodes["a"].Location,
subnet: kiloNet,
subnet: DefaultKiloSubnet,
privateIP: nodes["a"].InternalIP,
wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)},
segments: []*segment{
@ -148,7 +147,7 @@ func TestNewTopology(t *testing.T) {
hostname: nodes["b"].Name,
leader: true,
location: nodes["b"].Location,
subnet: kiloNet,
subnet: DefaultKiloSubnet,
privateIP: nodes["b"].InternalIP,
wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)},
segments: []*segment{
@ -184,7 +183,7 @@ func TestNewTopology(t *testing.T) {
hostname: nodes["c"].Name,
leader: false,
location: nodes["b"].Location,
subnet: kiloNet,
subnet: DefaultKiloSubnet,
privateIP: nodes["c"].InternalIP,
wireGuardCIDR: nil,
segments: []*segment{
@ -220,7 +219,7 @@ func TestNewTopology(t *testing.T) {
hostname: nodes["a"].Name,
leader: true,
location: nodes["a"].Name,
subnet: kiloNet,
subnet: DefaultKiloSubnet,
privateIP: nodes["a"].InternalIP,
wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)},
segments: []*segment{
@ -266,7 +265,7 @@ func TestNewTopology(t *testing.T) {
hostname: nodes["b"].Name,
leader: true,
location: nodes["b"].Name,
subnet: kiloNet,
subnet: DefaultKiloSubnet,
privateIP: nodes["b"].InternalIP,
wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)},
segments: []*segment{
@ -312,7 +311,7 @@ func TestNewTopology(t *testing.T) {
hostname: nodes["c"].Name,
leader: true,
location: nodes["c"].Name,
subnet: kiloNet,
subnet: DefaultKiloSubnet,
privateIP: nodes["c"].InternalIP,
wireGuardCIDR: &net.IPNet{IP: w3, Mask: net.CIDRMask(16, 32)},
segments: []*segment{
@ -353,7 +352,7 @@ func TestNewTopology(t *testing.T) {
} {
tc.result.key = key
tc.result.port = port
topo, err := NewTopology(nodes, peers, tc.granularity, tc.hostname, port, key, kiloNet)
topo, err := NewTopology(nodes, peers, tc.granularity, tc.hostname, port, key, DefaultKiloSubnet)
if err != nil {
t.Errorf("test case %q: failed to generate Topology: %v", tc.name, err)
}
@ -372,12 +371,12 @@ func mustTopo(t *testing.T, nodes map[string]*Node, peers map[string]*Peer, gran
}
func TestRoutes(t *testing.T) {
nodes, peers, key, port, kiloNet := setup(t)
nodes, peers, key, port := setup(t)
kiloIface := 0
privIface := 1
pubIface := 2
mustTopoForGranularityAndHost := func(granularity Granularity, hostname string) *Topology {
return mustTopo(t, nodes, peers, granularity, hostname, port, key, kiloNet)
return mustTopo(t, nodes, peers, granularity, hostname, port, key, DefaultKiloSubnet)
}
for _, tc := range []struct {
@ -987,7 +986,7 @@ func TestRoutes(t *testing.T) {
}
func TestConf(t *testing.T) {
nodes, peers, key, port, kiloNet := setup(t)
nodes, peers, key, port := setup(t)
for _, tc := range []struct {
name string
topology *Topology
@ -995,7 +994,7 @@ func TestConf(t *testing.T) {
}{
{
name: "logical from a",
topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["a"].Name, port, key, kiloNet),
topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["a"].Name, port, key, DefaultKiloSubnet),
result: `[Interface]
PrivateKey = private
ListenPort = 51820
@ -1019,7 +1018,7 @@ AllowedIPs = 10.5.0.3/24
},
{
name: "logical from b",
topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["b"].Name, port, key, kiloNet),
topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet),
result: `[Interface]
PrivateKey = private
ListenPort = 51820
@ -1043,7 +1042,7 @@ AllowedIPs = 10.5.0.3/24
},
{
name: "logical from c",
topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["c"].Name, port, key, kiloNet),
topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["c"].Name, port, key, DefaultKiloSubnet),
result: `[Interface]
PrivateKey = private
ListenPort = 51820
@ -1067,7 +1066,7 @@ AllowedIPs = 10.5.0.3/24
},
{
name: "full from a",
topology: mustTopo(t, nodes, peers, FullGranularity, nodes["a"].Name, port, key, kiloNet),
topology: mustTopo(t, nodes, peers, FullGranularity, nodes["a"].Name, port, key, DefaultKiloSubnet),
result: `[Interface]
PrivateKey = private
ListenPort = 51820
@ -1096,7 +1095,7 @@ AllowedIPs = 10.5.0.3/24
},
{
name: "full from b",
topology: mustTopo(t, nodes, peers, FullGranularity, nodes["b"].Name, port, key, kiloNet),
topology: mustTopo(t, nodes, peers, FullGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet),
result: `[Interface]
PrivateKey = private
ListenPort = 51820
@ -1125,7 +1124,7 @@ AllowedIPs = 10.5.0.3/24
},
{
name: "full from c",
topology: mustTopo(t, nodes, peers, FullGranularity, nodes["c"].Name, port, key, kiloNet),
topology: mustTopo(t, nodes, peers, FullGranularity, nodes["c"].Name, port, key, DefaultKiloSubnet),
result: `[Interface]
PrivateKey = private
ListenPort = 51820