*: add complete CNI support

This commit enables Kilo to work as an independent networking provider.
This is done by leveraging CNI. Kilo brings the necessary CNI plugins to
operate and takes care of all networking.

Add-on compatibility for Calico, Flannel, etc, will be re-introduced
shortly.

pkg/mesh/cni.go

@@ -0,0 +1,142 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mesh
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+
+	"github.com/containernetworking/cni/libcni"
+	"github.com/containernetworking/cni/pkg/types"
+	ipamallocator "github.com/containernetworking/plugins/plugins/ipam/host-local/backend/allocator"
+	"github.com/go-kit/kit/log/level"
+)
+
+// updateCNIConfig will try to update the local node's CNI config.
+func (m *Mesh) updateCNIConfig() {
+	m.mu.Lock()
+	n := m.nodes[m.hostname]
+	m.mu.Unlock()
+	if n == nil || n.Subnet == nil {
+		level.Debug(m.logger).Log("msg", "local node does not have a valid subnet assigned")
+		return
+	}
+
+	cidr, err := getCIDRFromCNI(m.cniPath)
+	if err != nil {
+		level.Warn(m.logger).Log("msg", "failed to get CIDR from CNI file; overwriting it", "err", err.Error())
+	}
+
+	if ipNetsEqual(cidr, n.Subnet) {
+		return
+	}
+
+	if cidr == nil {
+		level.Info(m.logger).Log("msg", "CIDR in CNI file is empty")
+	} else {
+		level.Info(m.logger).Log("msg", "CIDR in CNI file is not empty; overwriting", "old", cidr.String(), "new", n.Subnet.String())
+	}
+
+	if err := setCIDRInCNI(m.cniPath, n.Subnet); err != nil {
+		level.Warn(m.logger).Log("msg", "failed to set CIDR in CNI file", "err", err.Error())
+	}
+}
+
+// getCIDRFromCNI finds the CIDR for the node from the CNI configuration file.
+func getCIDRFromCNI(path string) (*net.IPNet, error) {
+	var cidr net.IPNet
+	var ic *ipamallocator.IPAMConfig
+
+	cl, err := libcni.ConfListFromFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CNI config list file: %v", err)
+	}
+	for _, conf := range cl.Plugins {
+		if conf.Network.IPAM.Type != "" {
+			ic, _, err = ipamallocator.LoadIPAMConfig(conf.Bytes, "")
+			if err != nil {
+				return nil, fmt.Errorf("failed to read IPAM config from CNI config list file: %v", err)
+			}
+			for _, set := range ic.Ranges {
+				for _, r := range set {
+					cidr = net.IPNet(r.Subnet)
+					if (&cidr).String() == "" {
+						continue
+					}
+					// Return the first subnet we find.
+					return &cidr, nil
+				}
+			}
+		}
+	}
+	return nil, nil
+}
+
+// setCIDRInCNI sets the CIDR allocated to the node in the CNI configuration file.
+func setCIDRInCNI(path string, cidr *net.IPNet) error {
+	f, err := ioutil.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read CNI config list file: %v", err)
+	}
+	raw := make(map[string]interface{})
+	if err := json.Unmarshal(f, &raw); err != nil {
+		return fmt.Errorf("failed to parse CNI config file: %v", err)
+	}
+	if _, ok := raw["plugins"]; !ok {
+		return errors.New("failed to find plugins in CNI config file")
+	}
+	plugins, ok := raw["plugins"].([]interface{})
+	if !ok {
+		return errors.New("failed to parse plugins in CNI config file")
+	}
+
+	var found bool
+	for i := range plugins {
+		p, ok := plugins[i].(map[string]interface{})
+		if !ok {
+			return fmt.Errorf("failed to parse plugin %d in CNI config file", i)
+		}
+		if _, ok := p["ipam"]; !ok {
+			continue
+		}
+		ipam, ok := p["ipam"].(map[string]interface{})
+		if !ok {
+			return errors.New("failed to parse IPAM configuration in CNI config file")
+		}
+		ipam["ranges"] = []ipamallocator.RangeSet{
+			{
+				{
+					Subnet: types.IPNet(*cidr),
+				},
+			},
+		}
+		found = true
+	}
+
+	if !found {
+		return errors.New("failed to set subnet CIDR in CNI config file; file appears invalid")
+	}
+	buf, err := json.Marshal(raw)
+	if err != nil {
+		return fmt.Errorf("failed to marshal CNI config: %v", err)
+	}
+	if err := ioutil.WriteFile(path, buf, 0644); err != nil {
+		return fmt.Errorf("failed to write CNI config file to disk: %v", err)
+	}
+	return nil
+}

pkg/mesh/mesh.go

@@ -29,7 +29,6 @@ import (
 	"github.com/vishvananda/netlink"
 
 	"github.com/squat/kilo/pkg/iproute"
-	"github.com/squat/kilo/pkg/ipset"
 	"github.com/squat/kilo/pkg/iptables"
 	"github.com/squat/kilo/pkg/route"
 	"github.com/squat/kilo/pkg/wireguard"
@@ -46,6 +45,8 @@ const (
 	ConfPath = KiloPath + "/conf"
 	// DefaultKiloPort is the default UDP port Kilo uses.
 	DefaultKiloPort = 51820
+	// DefaultCNIPath is the default path to the CNI config file.
+	DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist"
 )
 
 // Granularity represents the abstraction level at which the network
@@ -171,12 +172,13 @@ type PeerBackend interface {
 // Mesh is able to create Kilo network meshes.
 type Mesh struct {
 	Backend
+	cni         bool
+	cniPath     string
 	encapsulate Encapsulate
 	externalIP  *net.IPNet
 	granularity Granularity
 	hostname    string
 	internalIP  *net.IPNet
-	ipset       *ipset.Set
 	ipTables    *iptables.Controller
 	kiloIface   int
 	key         []byte
@@ -205,7 +207,7 @@ type Mesh struct {
 }
 
 // New returns a new Mesh instance.
-func New(backend Backend, encapsulate Encapsulate, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local bool, logger log.Logger) (*Mesh, error) {
+func New(backend Backend, encapsulate Encapsulate, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local, cni bool, cniPath string, logger log.Logger) (*Mesh, error) {
 	if err := os.MkdirAll(KiloPath, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create directory to store configuration: %v", err)
 	}
@@ -259,28 +261,27 @@ func New(backend Backend, encapsulate Encapsulate, granularity Granularity, host
 	}
 	return &Mesh{
 		Backend:     backend,
+		cni:         cni,
+		cniPath:     cniPath,
 		encapsulate: encapsulate,
 		externalIP:  publicIP,
 		granularity: granularity,
 		hostname:    hostname,
 		internalIP:  privateIP,
-		// This is a patch until Calico supports
-		// other hosts adding IPIP iptables rules.
-		ipset:     ipset.New("cali40all-hosts-net"),
-		ipTables:  ipTables,
-		kiloIface: kiloIface,
-		nodes:     make(map[string]*Node),
-		peers:     make(map[string]*Peer),
-		port:      port,
-		priv:      private,
-		privIface: privIface,
-		pub:       public,
-		pubIface:  pubIface,
-		local:     local,
-		stop:      make(chan struct{}),
-		subnet:    subnet,
-		table:     route.NewTable(),
-		tunlIface: tunlIface,
+		ipTables:    ipTables,
+		kiloIface:   kiloIface,
+		nodes:       make(map[string]*Node),
+		peers:       make(map[string]*Peer),
+		port:        port,
+		priv:        private,
+		privIface:   privIface,
+		pub:         public,
+		pubIface:    pubIface,
+		local:       local,
+		stop:        make(chan struct{}),
+		subnet:      subnet,
+		table:       route.NewTable(),
+		tunlIface:   tunlIface,
 		errorCounter: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Name: "kilo_errors_total",
 			Help: "Number of errors that occurred while administering the mesh.",
@@ -309,10 +310,6 @@ func (m *Mesh) Run() error {
 	if err := m.Peers().Init(m.stop); err != nil {
 		return fmt.Errorf("failed to initialize peer backend: %v", err)
 	}
-	ipsetErrors, err := m.ipset.Run(m.stop)
-	if err != nil {
-		return fmt.Errorf("failed to watch for ipset updates: %v", err)
-	}
 	ipTablesErrors, err := m.ipTables.Run(m.stop)
 	if err != nil {
 		return fmt.Errorf("failed to watch for IP tables updates: %v", err)
@@ -325,7 +322,6 @@ func (m *Mesh) Run() error {
 		for {
 			var err error
 			select {
-			case err = <-ipsetErrors:
 			case err = <-ipTablesErrors:
 			case err = <-routeErrors:
 			case <-m.stop:
@@ -351,6 +347,9 @@ func (m *Mesh) Run() error {
 			m.syncPeers(pe)
 		case <-t.C:
 			m.checkIn()
+			if m.cni {
+				m.updateCNIConfig()
+			}
 			m.syncEndpoints()
 			m.applyTopology()
 			t.Reset(resyncPeriod)
@@ -585,45 +584,40 @@ func (m *Mesh) applyTopology() {
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	var private *net.IPNet
-	// If we are not encapsulating packets to the local private network,
-	// then pass the private IP to add an exception to the NAT rule.
-	if m.encapsulate != AlwaysEncapsulate {
-		private = t.privateIP
-	}
-	rules := iptables.MasqueradeRules(private, m.nodes[m.hostname].Subnet, t.RemoteSubnets())
-	rules = append(rules, iptables.ForwardRules(m.subnet)...)
+	rules := iptables.ForwardRules(m.subnet)
+	var peerCIDRs []*net.IPNet
 	for _, p := range m.peers {
 		rules = append(rules, iptables.ForwardRules(p.AllowedIPs...)...)
+		peerCIDRs = append(peerCIDRs, p.AllowedIPs...)
+	}
+	rules = append(rules, iptables.MasqueradeRules(m.subnet, oneAddressCIDR(t.privateIP.IP), m.nodes[m.hostname].Subnet, t.RemoteSubnets(), peerCIDRs)...)
+	// If we are handling local routes, ensure the local
+	// tunnel has an IP address and IPIP traffic is allowed.
+	if m.encapsulate != NeverEncapsulate && m.local {
+		var cidrs []*net.IPNet
+		for _, s := range t.segments {
+			if s.location == m.nodes[m.hostname].Location {
+				for i := range s.privateIPs {
+					cidrs = append(cidrs, oneAddressCIDR(s.privateIPs[i]))
+				}
+				break
+			}
+		}
+		rules = append(rules, iptables.EncapsulateRules(cidrs)...)
+
+		// If we are handling local routes, ensure the local
+		// tunnel has an IP address.
+		if err := iproute.SetAddress(m.tunlIface, oneAddressCIDR(newAllocator(*m.nodes[m.hostname].Subnet).next().IP)); err != nil {
+			level.Error(m.logger).Log("error", err)
+			m.errorCounter.WithLabelValues("apply").Inc()
+			return
+		}
 	}
 	if err := m.ipTables.Set(rules); err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	if m.encapsulate != NeverEncapsulate {
-		var peers []net.IP
-		for _, s := range t.segments {
-			if s.location == m.nodes[m.hostname].Location {
-				peers = s.privateIPs
-				break
-			}
-		}
-		if err := m.ipset.Set(peers); err != nil {
-			level.Error(m.logger).Log("error", err)
-			m.errorCounter.WithLabelValues("apply").Inc()
-			return
-		}
-		// If we are handling local routes, ensure the local
-		// tunnel has an IP address.
-		if m.local {
-			if err := iproute.SetAddress(m.tunlIface, oneAddressCIDR(newAllocator(*m.nodes[m.hostname].Subnet).next().IP)); err != nil {
-				level.Error(m.logger).Log("error", err)
-				m.errorCounter.WithLabelValues("apply").Inc()
-				return
-			}
-		}
-	}
 	if t.leader {
 		if err := iproute.SetAddress(m.kiloIface, t.wireGuardCIDR); err != nil {
 			level.Error(m.logger).Log("error", err)
@@ -720,10 +714,6 @@ func (m *Mesh) cleanUp() {
 		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up peer backend: %v", err))
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}
-	if err := m.ipset.CleanUp(); err != nil {
-		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up ipset: %v", err))
-		m.errorCounter.WithLabelValues("cleanUp").Inc()
-	}
 }
 
 func isSelf(hostname string, node *Node) bool {
@@ -781,6 +771,12 @@ func ipNetsEqual(a, b *net.IPNet) bool {
 }
 
 func subnetsEqual(a, b *net.IPNet) bool {
+	if a == nil && b == nil {
+		return true
+	}
+	if (a != nil) != (b != nil) {
+		return false
+	}
 	if a.Mask.String() != b.Mask.String() {
 		return false
 	}

pkg/mesh/topology.go

@@ -51,7 +51,6 @@ type Topology struct {
 }
 
 type segment struct {
-	// Some fields need to be exported so that the template can read them.
 	allowedIPs []*net.IPNet
 	endpoint   net.IP
 	key        []byte