manifests,pkg/encapsulation: Flannel compatibility

This commit adds basic support to run in compatibility mode with
Flannel. This allows clusters running Flannel as their principal
networking solution to leverage some advances Kilo features. In certain
Flannel setups, the clusters can even leverage muti-cloud. For this, the
cluster needs to either run in a full mesh, or Flannel needs to use the
API server's external IP address.

pkg/encapsulation/encapsulation.go

@@ -0,0 +1,55 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encapsulation
+
+import (
+	"net"
+
+	"github.com/squat/kilo/pkg/iptables"
+)
+
+// Strategy identifies which packets within a location should
+// be encapsulated.
+type Strategy string
+
+const (
+	// Never indicates that no packets within a location
+	// should be encapsulated.
+	Never Strategy = "never"
+	// CrossSubnet indicates that only packets that
+	// traverse subnets within a location should be encapsulated.
+	CrossSubnet Strategy = "crosssubnet"
+	// Always indicates that all packets within a location
+	// should be encapsulated.
+	Always Strategy = "always"
+)
+
+// Encapsulator can:
+// * configure the encapsulation interface;
+// * determine the gateway IP corresponding to a node;
+// * get the encapsulation interface index;
+// * set the interface IP address;
+// * return the required IPTables rules;
+// * return the encapsulation strategy; and
+// * clean up any changes applied to the backend.
+type Encapsulator interface {
+	CleanUp() error
+	Gw(net.IP, net.IP, *net.IPNet) net.IP
+	Index() int
+	Init(int) error
+	Rules([]*net.IPNet) []iptables.Rule
+	Set(*net.IPNet) error
+	Strategy() Strategy
+}

pkg/encapsulation/flannel.go

@@ -0,0 +1,108 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encapsulation
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/squat/kilo/pkg/iptables"
+	"github.com/vishvananda/netlink"
+)
+
+const flannelDeviceName = "flannel.1"
+
+type flannel struct {
+	iface    int
+	strategy Strategy
+	ch       chan netlink.LinkUpdate
+	done     chan struct{}
+	// mu guards updates to the iface field.
+	mu sync.Mutex
+}
+
+// NewFlannel returns an encapsulator that uses Flannel.
+func NewFlannel(strategy Strategy) Encapsulator {
+	return &flannel{
+		ch:       make(chan netlink.LinkUpdate),
+		done:     make(chan struct{}),
+		strategy: strategy,
+	}
+}
+
+// CleanUp is a no-op.
+func (f *flannel) CleanUp() error {
+	close(f.done)
+	return nil
+}
+
+// Gw returns the correct gateway IP associated with the given node.
+func (f *flannel) Gw(_, _ net.IP, subnet *net.IPNet) net.IP {
+	return subnet.IP
+}
+
+// Index returns the index of the Flannel interface.
+func (f *flannel) Index() int {
+	return f.iface
+}
+
+// Init finds the Flannel interface index.
+func (f *flannel) Init(_ int) error {
+	if err := netlink.LinkSubscribe(f.ch, f.done); err != nil {
+		return fmt.Errorf("failed to subscribe to updates to %s: %v", flannelDeviceName, err)
+	}
+	go func() {
+		var lu netlink.LinkUpdate
+		for {
+			select {
+			case lu = <-f.ch:
+				if lu.Attrs().Name == flannelDeviceName {
+					f.mu.Lock()
+					f.iface = lu.Attrs().Index
+					f.mu.Unlock()
+				}
+			case <-f.done:
+				return
+			}
+		}
+	}()
+	i, err := netlink.LinkByName(flannelDeviceName)
+	if _, ok := err.(netlink.LinkNotFoundError); ok {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to query for Flannel interface: %v", err)
+	}
+	f.mu.Lock()
+	f.iface = i.Attrs().Index
+	f.mu.Unlock()
+	return nil
+}
+
+// Rules is a no-op.
+func (f *flannel) Rules(_ []*net.IPNet) []iptables.Rule {
+	return nil
+}
+
+// Set is a no-op.
+func (f *flannel) Set(_ *net.IPNet) error {
+	return nil
+}
+
+// Strategy returns the configured strategy for encapsulation.
+func (f *flannel) Strategy() Strategy {
+	return f.strategy
+}

pkg/encapsulation/ipip.go

@@ -22,45 +22,13 @@ import (
 	"github.com/squat/kilo/pkg/iptables"
 )
 
-// Strategy identifies which packets within a location should
-// be encapsulated.
-type Strategy string
-
-const (
-	// Never indicates that no packets within a location
-	// should be encapsulated.
-	Never Strategy = "never"
-	// CrossSubnet indicates that only packets that
-	// traverse subnets within a location should be encapsulated.
-	CrossSubnet Strategy = "crosssubnet"
-	// Always indicates that all packets within a location
-	// should be encapsulated.
-	Always Strategy = "always"
-)
-
-// Interface can configure
-// the encapsulation interface, init itself,
-// get the encapsulation interface index,
-// set the interface IP address,
-// return the required IPTables rules,
-// return the encapsulation strategy,
-// and clean up any changes applied to the backend.
-type Interface interface {
-	CleanUp() error
-	Index() int
-	Init(int) error
-	Rules([]*net.IPNet) []iptables.Rule
-	Set(*net.IPNet) error
-	Strategy() Strategy
-}
-
 type ipip struct {
 	iface    int
 	strategy Strategy
 }
 
-// NewIPIP returns an encapsulation that uses IPIP.
-func NewIPIP(strategy Strategy) Interface {
+// NewIPIP returns an encapsulator that uses IPIP.
+func NewIPIP(strategy Strategy) Encapsulator {
 	return &ipip{strategy: strategy}
 }
 
@@ -72,6 +40,11 @@ func (i *ipip) CleanUp() error {
 	return iproute.RemoveInterface(i.iface)
 }
 
+// Gw returns the correct gateway IP associated with the given node.
+func (i *ipip) Gw(_, internal net.IP, _ *net.IPNet) net.IP {
+	return internal
+}
+
 // Index returns the index of the IPIP interface.
 func (i *ipip) Index() int {
 	return i.iface

pkg/mesh/mesh.go

@@ -169,7 +169,7 @@ type Mesh struct {
 	Backend
 	cni         bool
 	cniPath     string
-	enc         encapsulation.Interface
+	enc         encapsulation.Encapsulator
 	externalIP  *net.IPNet
 	granularity Granularity
 	hostname    string
@@ -202,7 +202,7 @@ type Mesh struct {
 }
 
 // New returns a new Mesh instance.
-func New(backend Backend, enc encapsulation.Interface, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local, cni bool, cniPath string, logger log.Logger) (*Mesh, error) {
+func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local, cni bool, cniPath string, logger log.Logger) (*Mesh, error) {
 	if err := os.MkdirAll(KiloPath, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create directory to store configuration: %v", err)
 	}
@@ -245,7 +245,7 @@ func New(backend Backend, enc encapsulation.Interface, granularity Granularity,
 	}
 	if enc.Strategy() != encapsulation.Never {
 		if err := enc.Init(privIface); err != nil {
-			return nil, fmt.Errorf("failed to initialize encapsulation: %v", err)
+			return nil, fmt.Errorf("failed to initialize encapsulator: %v", err)
 		}
 	}
 	level.Debug(logger).Log("msg", fmt.Sprintf("using %s as the private IP address", privateIP.String()))
@@ -674,7 +674,7 @@ func (m *Mesh) applyTopology() {
 	}
 	// We need to add routes last since they may depend
 	// on the WireGuard interface.
-	routes := t.Routes(m.kiloIface, m.privIface, m.enc.Index(), m.local, m.enc.Strategy())
+	routes := t.Routes(m.kiloIface, m.privIface, m.enc.Index(), m.local, m.enc)
 	if err := m.table.Set(routes); err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
@@ -723,7 +723,7 @@ func (m *Mesh) cleanUp() {
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}
 	if err := m.enc.CleanUp(); err != nil {
-		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up encapsulation: %v", err))
+		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up encapsulator: %v", err))
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}
 }

pkg/mesh/topology.go

@@ -172,14 +172,16 @@ func (t *Topology) RemoteSubnets() []*net.IPNet {
 }
 
 // Routes generates a slice of routes for a given Topology.
-func (t *Topology) Routes(kiloIface, privIface, tunlIface int, local bool, encapsulate encapsulation.Strategy) []*netlink.Route {
+func (t *Topology) Routes(kiloIface, privIface, tunlIface int, local bool, enc encapsulation.Encapsulator) []*netlink.Route {
 	var routes []*netlink.Route
 	if !t.leader {
-		// Find the leader for this segment.
-		var leader net.IP
+		// Find the GW for this segment.
+		// This will be the an IP of the leader.
+		// In an IPIP encapsulated mesh it is the leader's private IP.
+		var gw net.IP
 		for _, segment := range t.segments {
 			if segment.location == t.location {
-				leader = segment.privateIPs[segment.leader]
+				gw = enc.Gw(segment.endpoint, segment.privateIPs[segment.leader], segment.cidrs[segment.leader])
 				break
 			}
 		}
@@ -188,10 +190,10 @@ func (t *Topology) Routes(kiloIface, privIface, tunlIface int, local bool, encap
 			routes = append(routes, encapsulateRoute(&netlink.Route{
 				Dst:       oneAddressCIDR(segment.wireGuardIP),
 				Flags:     int(netlink.FLAG_ONLINK),
-				Gw:        leader,
+				Gw:        gw,
 				LinkIndex: privIface,
 				Protocol:  unix.RTPROT_STATIC,
-			}, encapsulate, t.privateIP, tunlIface))
+			}, enc.Strategy(), t.privateIP, tunlIface))
 			// Add routes for the current segment if local is true.
 			if segment.location == t.location {
 				if local {
@@ -206,7 +208,7 @@ func (t *Topology) Routes(kiloIface, privIface, tunlIface int, local bool, encap
 							Gw:        segment.privateIPs[i],
 							LinkIndex: privIface,
 							Protocol:  unix.RTPROT_STATIC,
-						}, encapsulate, t.privateIP, tunlIface))
+						}, enc.Strategy(), t.privateIP, tunlIface))
 					}
 				}
 				continue
@@ -216,20 +218,20 @@ func (t *Topology) Routes(kiloIface, privIface, tunlIface int, local bool, encap
 				routes = append(routes, encapsulateRoute(&netlink.Route{
 					Dst:       segment.cidrs[i],
 					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        leader,
+					Gw:        gw,
 					LinkIndex: privIface,
 					Protocol:  unix.RTPROT_STATIC,
-				}, encapsulate, t.privateIP, tunlIface))
+				}, enc.Strategy(), t.privateIP, tunlIface))
 				// Add routes to the private IPs of nodes in other segments.
 				// Number of CIDRs and private IPs always match so
 				// we can reuse the loop.
 				routes = append(routes, encapsulateRoute(&netlink.Route{
 					Dst:       oneAddressCIDR(segment.privateIPs[i]),
 					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        leader,
+					Gw:        gw,
 					LinkIndex: privIface,
 					Protocol:  unix.RTPROT_STATIC,
-				}, encapsulate, t.privateIP, tunlIface))
+				}, enc.Strategy(), t.privateIP, tunlIface))
 			}
 		}
 		// Add routes for the allowed IPs of peers.
@@ -238,10 +240,10 @@ func (t *Topology) Routes(kiloIface, privIface, tunlIface int, local bool, encap
 				routes = append(routes, encapsulateRoute(&netlink.Route{
 					Dst:       peer.AllowedIPs[i],
 					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        leader,
+					Gw:        gw,
 					LinkIndex: privIface,
 					Protocol:  unix.RTPROT_STATIC,
-				}, encapsulate, t.privateIP, tunlIface))
+				}, enc.Strategy(), t.privateIP, tunlIface))
 			}
 		}
 		return routes
@@ -261,7 +263,7 @@ func (t *Topology) Routes(kiloIface, privIface, tunlIface int, local bool, encap
 						Gw:        segment.privateIPs[i],
 						LinkIndex: privIface,
 						Protocol:  unix.RTPROT_STATIC,
-					}, encapsulate, t.privateIP, tunlIface))
+					}, enc.Strategy(), t.privateIP, tunlIface))
 				}
 			}
 			continue

pkg/mesh/topology_test.go

@@ -979,7 +979,7 @@ func TestRoutes(t *testing.T) {
 			},
 		},
 	} {
-		routes := tc.topology.Routes(kiloIface, privIface, pubIface, tc.local, encapsulation.Never)
+		routes := tc.topology.Routes(kiloIface, privIface, pubIface, tc.local, encapsulation.NewIPIP(encapsulation.Never))
 		if diff := pretty.Compare(routes, tc.result); diff != "" {
 			t.Errorf("test case %q: got diff: %v", tc.name, diff)
 		}