Merge pull request #92 from squat/non_linux

pkg/*: allow kgctl to compile for other OSes

.travis.yml

@@ -20,6 +20,7 @@ install: true
 
 script:
   - make
+  - make all-build
   - make clean
   - make unit
   - make lint

Dockerfile

@@ -11,5 +11,5 @@ LABEL maintainer="squat <lserven@gmail.com>"
 RUN echo -e "https://alpine.global.ssl.fastly.net/alpine/v3.12/main\nhttps://alpine.global.ssl.fastly.net/alpine/v3.12/community" > /etc/apk/repositories && \
     apk add --no-cache ipset iptables ip6tables wireguard-tools
 COPY --from=cni bridge host-local loopback portmap /opt/cni/bin/
-COPY bin/$GOARCH/kg /opt/bin/
+COPY bin/linux/$GOARCH/kg /opt/bin/
 ENTRYPOINT ["/opt/bin/kg"]

Makefile

@@ -1,10 +1,16 @@
 export GO111MODULE=on
 .PHONY: push container clean container-name container-latest push-latest fmt lint test unit vendor header generate client deepcopy informer lister openapi manifest manfest-latest manifest-annotate manifest manfest-latest manifest-annotate
 
-ARCH ?= amd64
+OS ?= $(shell go env GOOS)
+ARCH ?= $(shell go env GOARCH)
+ALL_OS := linux darwin windows
 ALL_ARCH := amd64 arm arm64
 DOCKER_ARCH := "amd64" "arm v7" "arm64 v8"
-BINS := $(addprefix bin/$(ARCH)/,kg kgctl)
+ifeq ($(OS),linux)
+    BINS := bin/$(OS)/$(ARCH)/kg bin/$(OS)/$(ARCH)/kgctl
+else
+    BINS := bin/$(OS)/$(ARCH)/kgctl
+endif
 PROJECT := kilo
 PKG := github.com/squat/$(PROJECT)
 REGISTRY ?= index.docker.io
@@ -39,7 +45,7 @@ BASE_IMAGE ?= alpine:3.12
 build: $(BINS)
 
 build-%:
-	@$(MAKE) --no-print-directory ARCH=$* build
+	@$(MAKE) --no-print-directory OS=$(word 1,$(subst -, ,$*)) ARCH=$(word 2,$(subst -, ,$*)) build
 
 container-latest-%:
 	@$(MAKE) --no-print-directory ARCH=$* container-latest
@@ -53,7 +59,7 @@ push-latest-%:
 push-%:
 	@$(MAKE) --no-print-directory ARCH=$* push
 
-all-build: $(addprefix build-, $(ALL_ARCH))
+all-build: $(foreach os, $(ALL_OS), $(addprefix build-$(os)-, $(ALL_ARCH)))
 
 all-container: $(addprefix container-, $(ALL_ARCH))
 
@@ -133,7 +139,7 @@ pkg/k8s/apis/kilo/v1alpha1/openapi_generated.go: pkg/k8s/apis/kilo/v1alpha1/type
 	go fmt $@
 
 $(BINS): $(SRC) go.mod
-	@mkdir -p bin/$(ARCH)
+	@mkdir -p bin/$(word 2,$(subst /, ,$*))/$(word 3,$(subst /, ,$*))
 	@echo "building: $@"
 	@docker run --rm \
 	    -u $$(id -u):$$(id -g) \
@@ -141,8 +147,8 @@ $(BINS): $(SRC) go.mod
 	    -w /$(PROJECT) \
 	    $(BUILD_IMAGE) \
 	    /bin/sh -c " \
-	        GOARCH=$(ARCH) \
+	        GOARCH=$(word 3,$(subst /, ,$*)) \
-	        GOOS=linux \
+	        GOOS=$(word 2,$(subst /, ,$*)) \
 	        GOCACHE=/$(PROJECT)/.cache \
 		CGO_ENABLED=0 \
 		go build -mod=vendor -o $@ \
@@ -201,9 +207,9 @@ header: .header
 		exit 1; \
 	fi
 
-tmp/help.txt: bin/$(ARCH)/kg
+tmp/help.txt: bin/$(OS)/$(ARCH)/kg
 	mkdir -p tmp
-	bin/$(ARCH)/kg --help 2>&1 | head -n -1 > $@
+	bin//$(OS)/$(ARCH)/kg --help 2>&1 | head -n -1 > $@
 
 docs/kg.md: $(EMBEDMD_BINARY) tmp/help.txt
 	$(EMBEDMD_BINARY) -w $@
@@ -224,7 +230,7 @@ website/build/index.html: website/docs/README.md
 	yarn --cwd website build
 
 container: .container-$(ARCH)-$(VERSION) container-name
-.container-$(ARCH)-$(VERSION): $(BINS) Dockerfile
+.container-$(ARCH)-$(VERSION): bin/linux/$(ARCH)/kg Dockerfile
 	@i=0; for a in $(ALL_ARCH); do [ "$$a" = $(ARCH) ] && break; i=$$((i+1)); done; \
 	ia=""; iv=""; \
 	j=0; for a in $(DOCKER_ARCH); do \

pkg/mesh/backend.go

@@ -0,0 +1,152 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mesh
+
+import (
+	"net"
+	"time"
+
+	"github.com/squat/kilo/pkg/wireguard"
+)
+
+const (
+	// resyncPeriod is how often the mesh checks state if no events have been received.
+	resyncPeriod = 30 * time.Second
+	// DefaultKiloInterface is the default iterface created and used by Kilo.
+	DefaultKiloInterface = "kilo0"
+	// DefaultKiloPort is the default UDP port Kilo uses.
+	DefaultKiloPort = 51820
+	// DefaultCNIPath is the default path to the CNI config file.
+	DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist"
+)
+
+// DefaultKiloSubnet is the default CIDR for Kilo.
+var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}}
+
+// Granularity represents the abstraction level at which the network
+// should be meshed.
+type Granularity string
+
+const (
+	// LogicalGranularity indicates that the network should create
+	// a mesh between logical locations, e.g. data-centers, but not between
+	// all nodes within a single location.
+	LogicalGranularity Granularity = "location"
+	// FullGranularity indicates that the network should create
+	// a mesh between every node.
+	FullGranularity Granularity = "full"
+)
+
+// Node represents a node in the network.
+type Node struct {
+	Endpoint   *wireguard.Endpoint
+	Key        []byte
+	InternalIP *net.IPNet
+	// LastSeen is a Unix time for the last time
+	// the node confirmed it was live.
+	LastSeen int64
+	// Leader is a suggestion to Kilo that
+	// the node wants to lead its segment.
+	Leader              bool
+	Location            string
+	Name                string
+	PersistentKeepalive int
+	Subnet              *net.IPNet
+	WireGuardIP         *net.IPNet
+}
+
+// Ready indicates whether or not the node is ready.
+func (n *Node) Ready() bool {
+	// Nodes that are not leaders will not have WireGuardIPs, so it is not required.
+	return n != nil && n.Endpoint != nil && !(n.Endpoint.IP == nil && n.Endpoint.DNS == "") && n.Endpoint.Port != 0 && n.Key != nil && n.InternalIP != nil && n.Subnet != nil && time.Now().Unix()-n.LastSeen < int64(resyncPeriod)*2/int64(time.Second)
+}
+
+// Peer represents a peer in the network.
+type Peer struct {
+	wireguard.Peer
+	Name string
+}
+
+// Ready indicates whether or not the peer is ready.
+// Peers can have empty endpoints because they may not have an
+// IP, for example if they are behind a NAT, and thus
+// will not declare their endpoint and instead allow it to be
+// discovered.
+func (p *Peer) Ready() bool {
+	return p != nil && p.AllowedIPs != nil && len(p.AllowedIPs) != 0 && p.PublicKey != nil
+}
+
+// EventType describes what kind of an action an event represents.
+type EventType string
+
+const (
+	// AddEvent represents an action where an item was added.
+	AddEvent EventType = "add"
+	// DeleteEvent represents an action where an item was removed.
+	DeleteEvent EventType = "delete"
+	// UpdateEvent represents an action where an item was updated.
+	UpdateEvent EventType = "update"
+)
+
+// NodeEvent represents an event concerning a node in the cluster.
+type NodeEvent struct {
+	Type EventType
+	Node *Node
+	Old  *Node
+}
+
+// PeerEvent represents an event concerning a peer in the cluster.
+type PeerEvent struct {
+	Type EventType
+	Peer *Peer
+	Old  *Peer
+}
+
+// Backend can create clients for all of the
+// primitive types that Kilo deals with, namely:
+// * nodes; and
+// * peers.
+type Backend interface {
+	Nodes() NodeBackend
+	Peers() PeerBackend
+}
+
+// NodeBackend can get nodes by name, init itself,
+// list the nodes that should be meshed,
+// set Kilo properties for a node,
+// clean up any changes applied to the backend,
+// and watch for changes to nodes.
+type NodeBackend interface {
+	CleanUp(string) error
+	Get(string) (*Node, error)
+	Init(<-chan struct{}) error
+	List() ([]*Node, error)
+	Set(string, *Node) error
+	Watch() <-chan *NodeEvent
+}
+
+// PeerBackend can get peers by name, init itself,
+// list the peers that should be in the mesh,
+// set fields for a peer,
+// clean up any changes applied to the backend,
+// and watch for changes to peers.
+type PeerBackend interface {
+	CleanUp(string) error
+	Get(string) (*Peer, error)
+	Init(<-chan struct{}) error
+	List() ([]*Peer, error)
+	Set(string, *Peer) error
+	Watch() <-chan *PeerEvent
+}

pkg/mesh/cni.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build linux
+
 package mesh
 
 import (

pkg/mesh/discoverips.go

@@ -0,0 +1,288 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package mesh
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"sort"
+
+	"github.com/vishvananda/netlink"
+)
+
+// getIP returns a private and public IP address for the local node.
+// It selects the private IP address in the following order:
+// - private IP to which hostname resolves
+// - private IP assigned to interface of default route
+// - private IP assigned to local interface
+// - public IP to which hostname resolves
+// - public IP assigned to interface of default route
+// - public IP assigned to local interface
+// It selects the public IP address in the following order:
+// - public IP to which hostname resolves
+// - public IP assigned to interface of default route
+// - public IP assigned to local interface
+// - private IP to which hostname resolves
+// - private IP assigned to interface of default route
+// - private IP assigned to local interface
+// - if no IP was found, return nil and an error.
+func getIP(hostname string, ignoreIfaces ...int) (*net.IPNet, *net.IPNet, error) {
+	ignore := make(map[string]struct{})
+	for i := range ignoreIfaces {
+		if ignoreIfaces[i] == 0 {
+			// Only ignore valid interfaces.
+			continue
+		}
+		iface, err := net.InterfaceByIndex(ignoreIfaces[i])
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to find interface %d: %v", ignoreIfaces[i], err)
+		}
+		ips, err := ipsForInterface(iface)
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, ip := range ips {
+			ignore[ip.String()] = struct{}{}
+			ignore[oneAddressCIDR(ip.IP).String()] = struct{}{}
+		}
+	}
+	var hostPriv, hostPub []*net.IPNet
+	{
+		// Check IPs to which hostname resolves first.
+		ips := ipsForHostname(hostname)
+		for _, ip := range ips {
+			ok, mask, err := assignedToInterface(ip)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to search locally assigned addresses: %v", err)
+			}
+			if !ok {
+				continue
+			}
+			ip.Mask = mask
+			if isPublic(ip.IP) {
+				hostPub = append(hostPub, ip)
+				continue
+			}
+			hostPriv = append(hostPriv, ip)
+		}
+		sortIPs(hostPriv)
+		sortIPs(hostPub)
+	}
+
+	var defaultPriv, defaultPub []*net.IPNet
+	{
+		// Check IPs on interface for default route next.
+		iface, err := defaultInterface()
+		if err != nil {
+			return nil, nil, err
+		}
+		ips, err := ipsForInterface(iface)
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, ip := range ips {
+			if isLocal(ip.IP) {
+				continue
+			}
+			if isPublic(ip.IP) {
+				defaultPub = append(defaultPub, ip)
+				continue
+			}
+			defaultPriv = append(defaultPriv, ip)
+		}
+		sortIPs(defaultPriv)
+		sortIPs(defaultPub)
+	}
+
+	var interfacePriv, interfacePub []*net.IPNet
+	{
+		// Finally look for IPs on all interfaces.
+		ips, err := ipsForAllInterfaces()
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, ip := range ips {
+			if isLocal(ip.IP) {
+				continue
+			}
+			if isPublic(ip.IP) {
+				interfacePub = append(interfacePub, ip)
+				continue
+			}
+			interfacePriv = append(interfacePriv, ip)
+		}
+		sortIPs(interfacePriv)
+		sortIPs(interfacePub)
+	}
+
+	var priv, pub, tmpPriv, tmpPub []*net.IPNet
+	tmpPriv = append(tmpPriv, hostPriv...)
+	tmpPriv = append(tmpPriv, defaultPriv...)
+	tmpPriv = append(tmpPriv, interfacePriv...)
+	tmpPub = append(tmpPub, hostPub...)
+	tmpPub = append(tmpPub, defaultPub...)
+	tmpPub = append(tmpPub, interfacePub...)
+	for i := range tmpPriv {
+		if _, ok := ignore[tmpPriv[i].String()]; ok {
+			continue
+		}
+		priv = append(priv, tmpPriv[i])
+	}
+	for i := range tmpPub {
+		if _, ok := ignore[tmpPub[i].String()]; ok {
+			continue
+		}
+		pub = append(pub, tmpPub[i])
+	}
+	if len(priv) == 0 && len(pub) == 0 {
+		return nil, nil, errors.New("no valid IP was found")
+	}
+	if len(priv) == 0 {
+		priv = pub
+	}
+	if len(pub) == 0 {
+		pub = priv
+	}
+	return priv[0], pub[0], nil
+}
+
+func assignedToInterface(ip *net.IPNet) (bool, net.IPMask, error) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return false, nil, fmt.Errorf("failed to list interfaces: %v", err)
+	}
+	// Sort the links for stability.
+	sort.Slice(links, func(i, j int) bool {
+		return links[i].Attrs().Name < links[j].Attrs().Name
+	})
+	for _, link := range links {
+		addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+		if err != nil {
+			return false, nil, fmt.Errorf("failed to list addresses for %s: %v", link.Attrs().Name, err)
+		}
+		// Sort the IPs for stability.
+		sort.Slice(addrs, func(i, j int) bool {
+			return addrs[i].String() < addrs[j].String()
+		})
+		for i := range addrs {
+			if ip.IP.Equal(addrs[i].IP) {
+				return true, addrs[i].Mask, nil
+			}
+		}
+	}
+	return false, nil, nil
+}
+
+// ipsForHostname returns a slice of IPs to which the
+// given hostname resolves.
+func ipsForHostname(hostname string) []*net.IPNet {
+	if ip := net.ParseIP(hostname); ip != nil {
+		return []*net.IPNet{oneAddressCIDR(ip)}
+	}
+	ips, err := net.LookupIP(hostname)
+	if err != nil {
+		// Most likely the hostname is not resolvable.
+		return nil
+	}
+	nets := make([]*net.IPNet, len(ips))
+	for i := range ips {
+		nets[i] = oneAddressCIDR(ips[i])
+	}
+	return nets
+}
+
+// ipsForAllInterfaces returns a slice of IPs assigned to all the
+// interfaces on the host.
+func ipsForAllInterfaces() ([]*net.IPNet, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list interfaces: %v", err)
+	}
+	var nets []*net.IPNet
+	for _, iface := range ifaces {
+		ips, err := ipsForInterface(&iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
+		}
+		nets = append(nets, ips...)
+	}
+	return nets, nil
+}
+
+// ipsForInterface returns a slice of IPs assigned to the given interface.
+func ipsForInterface(iface *net.Interface) ([]*net.IPNet, error) {
+	link, err := netlink.LinkByIndex(iface.Index)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get link: %s", err)
+	}
+	addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
+	}
+	var ips []*net.IPNet
+	for _, a := range addrs {
+		if a.IPNet != nil {
+			ips = append(ips, a.IPNet)
+		}
+	}
+	return ips, nil
+}
+
+// interfacesForIP returns a slice of interfaces withthe given IP.
+func interfacesForIP(ip *net.IPNet) ([]net.Interface, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list interfaces: %v", err)
+	}
+	var interfaces []net.Interface
+	for _, iface := range ifaces {
+		ips, err := ipsForInterface(&iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
+		}
+		for i := range ips {
+			if ip.IP.Equal(ips[i].IP) {
+				interfaces = append(interfaces, iface)
+				break
+			}
+		}
+	}
+	if len(interfaces) == 0 {
+		return nil, fmt.Errorf("no interface has %s assigned", ip.String())
+	}
+	return interfaces, nil
+}
+
+// defaultInterface returns the interface for the default route of the host.
+func defaultInterface() (*net.Interface, error) {
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, route := range routes {
+		if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" || route.Dst.String() == "::/0" {
+			if route.LinkIndex <= 0 {
+				return nil, errors.New("failed to determine interface of route")
+			}
+			return net.InterfaceByIndex(route.LinkIndex)
+		}
+	}
+
+	return nil, errors.New("failed to find default route")
+}

pkg/mesh/ip.go

@@ -15,150 +15,10 @@
 package mesh
 
 import (
-	"errors"
-	"fmt"
 	"net"
 	"sort"
-
-	"github.com/vishvananda/netlink"
 )
 
-// getIP returns a private and public IP address for the local node.
-// It selects the private IP address in the following order:
-// - private IP to which hostname resolves
-// - private IP assigned to interface of default route
-// - private IP assigned to local interface
-// - public IP to which hostname resolves
-// - public IP assigned to interface of default route
-// - public IP assigned to local interface
-// It selects the public IP address in the following order:
-// - public IP to which hostname resolves
-// - public IP assigned to interface of default route
-// - public IP assigned to local interface
-// - private IP to which hostname resolves
-// - private IP assigned to interface of default route
-// - private IP assigned to local interface
-// - if no IP was found, return nil and an error.
-func getIP(hostname string, ignoreIfaces ...int) (*net.IPNet, *net.IPNet, error) {
-	ignore := make(map[string]struct{})
-	for i := range ignoreIfaces {
-		if ignoreIfaces[i] == 0 {
-			// Only ignore valid interfaces.
-			continue
-		}
-		iface, err := net.InterfaceByIndex(ignoreIfaces[i])
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to find interface %d: %v", ignoreIfaces[i], err)
-		}
-		ips, err := ipsForInterface(iface)
-		if err != nil {
-			return nil, nil, err
-		}
-		for _, ip := range ips {
-			ignore[ip.String()] = struct{}{}
-			ignore[oneAddressCIDR(ip.IP).String()] = struct{}{}
-		}
-	}
-	var hostPriv, hostPub []*net.IPNet
-	{
-		// Check IPs to which hostname resolves first.
-		ips := ipsForHostname(hostname)
-		for _, ip := range ips {
-			ok, mask, err := assignedToInterface(ip)
-			if err != nil {
-				return nil, nil, fmt.Errorf("failed to search locally assigned addresses: %v", err)
-			}
-			if !ok {
-				continue
-			}
-			ip.Mask = mask
-			if isPublic(ip.IP) {
-				hostPub = append(hostPub, ip)
-				continue
-			}
-			hostPriv = append(hostPriv, ip)
-		}
-		sortIPs(hostPriv)
-		sortIPs(hostPub)
-	}
-
-	var defaultPriv, defaultPub []*net.IPNet
-	{
-		// Check IPs on interface for default route next.
-		iface, err := defaultInterface()
-		if err != nil {
-			return nil, nil, err
-		}
-		ips, err := ipsForInterface(iface)
-		if err != nil {
-			return nil, nil, err
-		}
-		for _, ip := range ips {
-			if isLocal(ip.IP) {
-				continue
-			}
-			if isPublic(ip.IP) {
-				defaultPub = append(defaultPub, ip)
-				continue
-			}
-			defaultPriv = append(defaultPriv, ip)
-		}
-		sortIPs(defaultPriv)
-		sortIPs(defaultPub)
-	}
-
-	var interfacePriv, interfacePub []*net.IPNet
-	{
-		// Finally look for IPs on all interfaces.
-		ips, err := ipsForAllInterfaces()
-		if err != nil {
-			return nil, nil, err
-		}
-		for _, ip := range ips {
-			if isLocal(ip.IP) {
-				continue
-			}
-			if isPublic(ip.IP) {
-				interfacePub = append(interfacePub, ip)
-				continue
-			}
-			interfacePriv = append(interfacePriv, ip)
-		}
-		sortIPs(interfacePriv)
-		sortIPs(interfacePub)
-	}
-
-	var priv, pub, tmpPriv, tmpPub []*net.IPNet
-	tmpPriv = append(tmpPriv, hostPriv...)
-	tmpPriv = append(tmpPriv, defaultPriv...)
-	tmpPriv = append(tmpPriv, interfacePriv...)
-	tmpPub = append(tmpPub, hostPub...)
-	tmpPub = append(tmpPub, defaultPub...)
-	tmpPub = append(tmpPub, interfacePub...)
-	for i := range tmpPriv {
-		if _, ok := ignore[tmpPriv[i].String()]; ok {
-			continue
-		}
-		priv = append(priv, tmpPriv[i])
-	}
-	for i := range tmpPub {
-		if _, ok := ignore[tmpPub[i].String()]; ok {
-			continue
-		}
-		pub = append(pub, tmpPub[i])
-	}
-	if len(priv) == 0 && len(pub) == 0 {
-		return nil, nil, errors.New("no valid IP was found")
-	}
-	if len(priv) == 0 {
-		priv = pub
-	}
-	if len(pub) == 0 {
-		pub = priv
-	}
-	return priv[0], pub[0], nil
-}
-
 // sortIPs sorts IPs so the result is stable.
 // It will first sort IPs by type, to prefer selecting
 // IPs of the same type, and then by value.
@@ -175,33 +35,6 @@ func sortIPs(ips []*net.IPNet) {
 	})
 }
 
-func assignedToInterface(ip *net.IPNet) (bool, net.IPMask, error) {
-	links, err := netlink.LinkList()
-	if err != nil {
-		return false, nil, fmt.Errorf("failed to list interfaces: %v", err)
-	}
-	// Sort the links for stability.
-	sort.Slice(links, func(i, j int) bool {
-		return links[i].Attrs().Name < links[j].Attrs().Name
-	})
-	for _, link := range links {
-		addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
-		if err != nil {
-			return false, nil, fmt.Errorf("failed to list addresses for %s: %v", link.Attrs().Name, err)
-		}
-		// Sort the IPs for stability.
-		sort.Slice(addrs, func(i, j int) bool {
-			return addrs[i].String() < addrs[j].String()
-		})
-		for i := range addrs {
-			if ip.IP.Equal(addrs[i].IP) {
-				return true, addrs[i].Mask, nil
-			}
-		}
-	}
-	return false, nil, nil
-}
-
 func isLocal(ip net.IP) bool {
 	return ip.IsLoopback() || ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()
 }
@@ -236,105 +69,6 @@ func isPublic(ip net.IP) bool {
 	return false
 }
 
-// ipsForHostname returns a slice of IPs to which the
-// given hostname resolves.
-func ipsForHostname(hostname string) []*net.IPNet {
-	if ip := net.ParseIP(hostname); ip != nil {
-		return []*net.IPNet{oneAddressCIDR(ip)}
-	}
-	ips, err := net.LookupIP(hostname)
-	if err != nil {
-		// Most likely the hostname is not resolvable.
-		return nil
-	}
-	nets := make([]*net.IPNet, len(ips))
-	for i := range ips {
-		nets[i] = oneAddressCIDR(ips[i])
-	}
-	return nets
-}
-
-// ipsForAllInterfaces returns a slice of IPs assigned to all the
-// interfaces on the host.
-func ipsForAllInterfaces() ([]*net.IPNet, error) {
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list interfaces: %v", err)
-	}
-	var nets []*net.IPNet
-	for _, iface := range ifaces {
-		ips, err := ipsForInterface(&iface)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
-		}
-		nets = append(nets, ips...)
-	}
-	return nets, nil
-}
-
-// ipsForInterface returns a slice of IPs assigned to the given interface.
-func ipsForInterface(iface *net.Interface) ([]*net.IPNet, error) {
-	link, err := netlink.LinkByIndex(iface.Index)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get link: %s", err)
-	}
-	addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
-	if err != nil {
-		return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
-	}
-	var ips []*net.IPNet
-	for _, a := range addrs {
-		if a.IPNet != nil {
-			ips = append(ips, a.IPNet)
-		}
-	}
-	return ips, nil
-}
-
-// interfacesForIP returns a slice of interfaces withthe given IP.
-func interfacesForIP(ip *net.IPNet) ([]net.Interface, error) {
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list interfaces: %v", err)
-	}
-	var interfaces []net.Interface
-	for _, iface := range ifaces {
-		ips, err := ipsForInterface(&iface)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
-		}
-		for i := range ips {
-			if ip.IP.Equal(ips[i].IP) {
-				interfaces = append(interfaces, iface)
-				break
-			}
-		}
-	}
-	if len(interfaces) == 0 {
-		return nil, fmt.Errorf("no interface has %s assigned", ip.String())
-	}
-	return interfaces, nil
-}
-
-// defaultInterface returns the interface for the default route of the host.
-func defaultInterface() (*net.Interface, error) {
-	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, route := range routes {
-		if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" || route.Dst.String() == "::/0" {
-			if route.LinkIndex <= 0 {
-				return nil, errors.New("failed to determine interface of route")
-			}
-			return net.InterfaceByIndex(route.LinkIndex)
-		}
-	}
-
-	return nil, errors.New("failed to find default route")
-}
-
 type allocator struct {
 	bits    int
 	ones    int

pkg/mesh/mesh.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build linux
+
 package mesh
 
 import (
@@ -35,142 +37,15 @@ import (
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
-const resyncPeriod = 30 * time.Second
-
 const (
-	// KiloPath is the directory where Kilo stores its configuration.
+	// kiloPath is the directory where Kilo stores its configuration.
-	KiloPath = "/var/lib/kilo"
+	kiloPath = "/var/lib/kilo"
-	// PrivateKeyPath is the filepath where the WireGuard private key is stored.
+	// privateKeyPath is the filepath where the WireGuard private key is stored.
-	PrivateKeyPath = KiloPath + "/key"
+	privateKeyPath = kiloPath + "/key"
-	// ConfPath is the filepath where the WireGuard configuration is stored.
+	// confPath is the filepath where the WireGuard configuration is stored.
-	ConfPath = KiloPath + "/conf"
+	confPath = kiloPath + "/conf"
-	// DefaultKiloInterface is the default iterface created and used by Kilo.
-	DefaultKiloInterface = "kilo0"
-	// DefaultKiloPort is the default UDP port Kilo uses.
-	DefaultKiloPort = 51820
-	// DefaultCNIPath is the default path to the CNI config file.
-	DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist"
 )
 
-// DefaultKiloSubnet is the default CIDR for Kilo.
-var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}}
-
-// Granularity represents the abstraction level at which the network
-// should be meshed.
-type Granularity string
-
-const (
-	// LogicalGranularity indicates that the network should create
-	// a mesh between logical locations, e.g. data-centers, but not between
-	// all nodes within a single location.
-	LogicalGranularity Granularity = "location"
-	// FullGranularity indicates that the network should create
-	// a mesh between every node.
-	FullGranularity Granularity = "full"
-)
-
-// Node represents a node in the network.
-type Node struct {
-	Endpoint   *wireguard.Endpoint
-	Key        []byte
-	InternalIP *net.IPNet
-	// LastSeen is a Unix time for the last time
-	// the node confirmed it was live.
-	LastSeen int64
-	// Leader is a suggestion to Kilo that
-	// the node wants to lead its segment.
-	Leader              bool
-	Location            string
-	Name                string
-	PersistentKeepalive int
-	Subnet              *net.IPNet
-	WireGuardIP         *net.IPNet
-}
-
-// Ready indicates whether or not the node is ready.
-func (n *Node) Ready() bool {
-	// Nodes that are not leaders will not have WireGuardIPs, so it is not required.
-	return n != nil && n.Endpoint != nil && !(n.Endpoint.IP == nil && n.Endpoint.DNS == "") && n.Endpoint.Port != 0 && n.Key != nil && n.InternalIP != nil && n.Subnet != nil && time.Now().Unix()-n.LastSeen < int64(resyncPeriod)*2/int64(time.Second)
-}
-
-// Peer represents a peer in the network.
-type Peer struct {
-	wireguard.Peer
-	Name string
-}
-
-// Ready indicates whether or not the peer is ready.
-// Peers can have empty endpoints because they may not have an
-// IP, for example if they are behind a NAT, and thus
-// will not declare their endpoint and instead allow it to be
-// discovered.
-func (p *Peer) Ready() bool {
-	return p != nil && p.AllowedIPs != nil && len(p.AllowedIPs) != 0 && p.PublicKey != nil
-}
-
-// EventType describes what kind of an action an event represents.
-type EventType string
-
-const (
-	// AddEvent represents an action where an item was added.
-	AddEvent EventType = "add"
-	// DeleteEvent represents an action where an item was removed.
-	DeleteEvent EventType = "delete"
-	// UpdateEvent represents an action where an item was updated.
-	UpdateEvent EventType = "update"
-)
-
-// NodeEvent represents an event concerning a node in the cluster.
-type NodeEvent struct {
-	Type EventType
-	Node *Node
-	Old  *Node
-}
-
-// PeerEvent represents an event concerning a peer in the cluster.
-type PeerEvent struct {
-	Type EventType
-	Peer *Peer
-	Old  *Peer
-}
-
-// Backend can create clients for all of the
-// primitive types that Kilo deals with, namely:
-// * nodes; and
-// * peers.
-type Backend interface {
-	Nodes() NodeBackend
-	Peers() PeerBackend
-}
-
-// NodeBackend can get nodes by name, init itself,
-// list the nodes that should be meshed,
-// set Kilo properties for a node,
-// clean up any changes applied to the backend,
-// and watch for changes to nodes.
-type NodeBackend interface {
-	CleanUp(string) error
-	Get(string) (*Node, error)
-	Init(<-chan struct{}) error
-	List() ([]*Node, error)
-	Set(string, *Node) error
-	Watch() <-chan *NodeEvent
-}
-
-// PeerBackend can get peers by name, init itself,
-// list the peers that should be in the mesh,
-// set fields for a peer,
-// clean up any changes applied to the backend,
-// and watch for changes to peers.
-type PeerBackend interface {
-	CleanUp(string) error
-	Get(string) (*Peer, error)
-	Init(<-chan struct{}) error
-	List() ([]*Peer, error)
-	Set(string, *Peer) error
-	Watch() <-chan *PeerEvent
-}
-
 // Mesh is able to create Kilo network meshes.
 type Mesh struct {
 	Backend
@@ -211,10 +86,10 @@ type Mesh struct {
 
 // New returns a new Mesh instance.
 func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, logger log.Logger) (*Mesh, error) {
-	if err := os.MkdirAll(KiloPath, 0700); err != nil {
+	if err := os.MkdirAll(kiloPath, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create directory to store configuration: %v", err)
 	}
-	private, err := ioutil.ReadFile(PrivateKeyPath)
+	private, err := ioutil.ReadFile(privateKeyPath)
 	private = bytes.Trim(private, "\n")
 	if err != nil {
 		level.Warn(logger).Log("msg", "no private key found on disk; generating one now")
@@ -226,7 +101,7 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(PrivateKeyPath, private, 0600); err != nil {
+	if err := ioutil.WriteFile(privateKeyPath, private, 0600); err != nil {
 		return nil, fmt.Errorf("failed to write private key to disk: %v", err)
 	}
 	cniIndex, err := cniDeviceIndex()
@@ -589,7 +464,7 @@ func (m *Mesh) applyTopology() {
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	if err := ioutil.WriteFile(ConfPath, buf, 0600); err != nil {
+	if err := ioutil.WriteFile(confPath, buf, 0600); err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
@@ -636,7 +511,7 @@ func (m *Mesh) applyTopology() {
 		equal := conf.Equal(oldConf)
 		if !equal {
 			level.Info(m.logger).Log("msg", "WireGuard configurations are different")
-			if err := wireguard.SetConf(link.Attrs().Name, ConfPath); err != nil {
+			if err := wireguard.SetConf(link.Attrs().Name, confPath); err != nil {
 				level.Error(m.logger).Log("error", err)
 				m.errorCounter.WithLabelValues("apply").Inc()
 				return
@@ -691,7 +566,7 @@ func (m *Mesh) cleanUp() {
 		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up routes: %v", err))
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}
-	if err := os.Remove(ConfPath); err != nil {
+	if err := os.Remove(confPath); err != nil {
 		level.Error(m.logger).Log("error", fmt.Sprintf("failed to delete configuration file: %v", err))
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}

pkg/mesh/routes.go

@@ -0,0 +1,252 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package mesh
+
+import (
+	"net"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/squat/kilo/pkg/encapsulation"
+	"github.com/squat/kilo/pkg/iptables"
+)
+
+const kiloTableIndex = 1107
+
+// Routes generates a slice of routes for a given Topology.
+func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface int, local bool, enc encapsulation.Encapsulator) ([]*netlink.Route, []*netlink.Rule) {
+	var routes []*netlink.Route
+	var rules []*netlink.Rule
+	if !t.leader {
+		// Find the GW for this segment.
+		// This will be the an IP of the leader.
+		// In an IPIP encapsulated mesh it is the leader's private IP.
+		var gw net.IP
+		for _, segment := range t.segments {
+			if segment.location == t.location {
+				gw = enc.Gw(segment.endpoint.IP, segment.privateIPs[segment.leader], segment.cidrs[segment.leader])
+				break
+			}
+		}
+		for _, segment := range t.segments {
+			// First, add a route to the WireGuard IP of the segment.
+			routes = append(routes, encapsulateRoute(&netlink.Route{
+				Dst:       oneAddressCIDR(segment.wireGuardIP),
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        gw,
+				LinkIndex: privIface,
+				Protocol:  unix.RTPROT_STATIC,
+			}, enc.Strategy(), t.privateIP, tunlIface))
+			// Add routes for the current segment if local is true.
+			if segment.location == t.location {
+				if local {
+					for i := range segment.cidrs {
+						// Don't add routes for the local node.
+						if segment.privateIPs[i].Equal(t.privateIP.IP) {
+							continue
+						}
+						routes = append(routes, encapsulateRoute(&netlink.Route{
+							Dst:       segment.cidrs[i],
+							Flags:     int(netlink.FLAG_ONLINK),
+							Gw:        segment.privateIPs[i],
+							LinkIndex: privIface,
+							Protocol:  unix.RTPROT_STATIC,
+						}, enc.Strategy(), t.privateIP, tunlIface))
+						// Encapsulate packets from the host's Pod subnet headed
+						// to private IPs.
+						if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
+							routes = append(routes, &netlink.Route{
+								Dst:       oneAddressCIDR(segment.privateIPs[i]),
+								Flags:     int(netlink.FLAG_ONLINK),
+								Gw:        segment.privateIPs[i],
+								LinkIndex: tunlIface,
+								Protocol:  unix.RTPROT_STATIC,
+								Table:     kiloTableIndex,
+							})
+							rules = append(rules, defaultRule(&netlink.Rule{
+								Src:   t.subnet,
+								Dst:   oneAddressCIDR(segment.privateIPs[i]),
+								Table: kiloTableIndex,
+							}))
+						}
+					}
+				}
+				continue
+			}
+			for i := range segment.cidrs {
+				// Add routes to the Pod CIDRs of nodes in other segments.
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       segment.cidrs[i],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+				// Add routes to the private IPs of nodes in other segments.
+				// Number of CIDRs and private IPs always match so
+				// we can reuse the loop.
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       oneAddressCIDR(segment.privateIPs[i]),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
+		}
+		// Add routes for the allowed IPs of peers.
+		for _, peer := range t.peers {
+			for i := range peer.AllowedIPs {
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       peer.AllowedIPs[i],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
+		}
+		return routes, rules
+	}
+	for _, segment := range t.segments {
+		// Add routes for the current segment if local is true.
+		if segment.location == t.location {
+			if local {
+				for i := range segment.cidrs {
+					// Don't add routes for the local node.
+					if segment.privateIPs[i].Equal(t.privateIP.IP) {
+						continue
+					}
+					routes = append(routes, encapsulateRoute(&netlink.Route{
+						Dst:       segment.cidrs[i],
+						Flags:     int(netlink.FLAG_ONLINK),
+						Gw:        segment.privateIPs[i],
+						LinkIndex: privIface,
+						Protocol:  unix.RTPROT_STATIC,
+					}, enc.Strategy(), t.privateIP, tunlIface))
+					// Encapsulate packets from the host's Pod subnet headed
+					// to private IPs.
+					if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
+						routes = append(routes, &netlink.Route{
+							Dst:       oneAddressCIDR(segment.privateIPs[i]),
+							Flags:     int(netlink.FLAG_ONLINK),
+							Gw:        segment.privateIPs[i],
+							LinkIndex: tunlIface,
+							Protocol:  unix.RTPROT_STATIC,
+							Table:     kiloTableIndex,
+						})
+						rules = append(rules, defaultRule(&netlink.Rule{
+							Src:   t.subnet,
+							Dst:   oneAddressCIDR(segment.privateIPs[i]),
+							Table: kiloTableIndex,
+						}))
+						// Also encapsulate packets from the Kilo interface
+						// headed to private IPs.
+						rules = append(rules, defaultRule(&netlink.Rule{
+							Dst:     oneAddressCIDR(segment.privateIPs[i]),
+							Table:   kiloTableIndex,
+							IifName: kiloIfaceName,
+						}))
+					}
+				}
+			}
+			continue
+		}
+		for i := range segment.cidrs {
+			// Add routes to the Pod CIDRs of nodes in other segments.
+			routes = append(routes, &netlink.Route{
+				Dst:       segment.cidrs[i],
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+			// Don't add routes through Kilo if the private IP
+			// equals the external IP. This means that the node
+			// is only accessible through an external IP and we
+			// cannot encapsulate traffic to an IP through the IP.
+			if segment.privateIPs[i].Equal(segment.endpoint.IP) {
+				continue
+			}
+			// Add routes to the private IPs of nodes in other segments.
+			// Number of CIDRs and private IPs always match so
+			// we can reuse the loop.
+			routes = append(routes, &netlink.Route{
+				Dst:       oneAddressCIDR(segment.privateIPs[i]),
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+	}
+	// Add routes for the allowed IPs of peers.
+	for _, peer := range t.peers {
+		for i := range peer.AllowedIPs {
+			routes = append(routes, &netlink.Route{
+				Dst:       peer.AllowedIPs[i],
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+	}
+	return routes, rules
+}
+
+func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route {
+	if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) {
+		route.LinkIndex = tunlIface
+	}
+	return route
+}
+
+// Rules returns the iptables rules required by the local node.
+func (t *Topology) Rules(cni bool) []iptables.Rule {
+	var rules []iptables.Rule
+	rules = append(rules, iptables.NewIPv4Chain("nat", "KILO-NAT"))
+	rules = append(rules, iptables.NewIPv6Chain("nat", "KILO-NAT"))
+	if cni {
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(t.subnet.IP)), "nat", "POSTROUTING", "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-s", t.subnet.String(), "-j", "KILO-NAT"))
+	}
+	for _, s := range t.segments {
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(s.wireGuardIP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuared IPs", "-d", s.wireGuardIP.String(), "-j", "RETURN"))
+		for _, aip := range s.allowedIPs {
+			rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-d", aip.String(), "-j", "RETURN"))
+		}
+	}
+	for _, p := range t.peers {
+		for _, aip := range p.AllowedIPs {
+			rules = append(rules,
+				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "POSTROUTING", "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-s", aip.String(), "-j", "KILO-NAT"),
+				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for peers", "-d", aip.String(), "-j", "RETURN"),
+			)
+		}
+	}
+	rules = append(rules, iptables.NewIPv4Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
+	rules = append(rules, iptables.NewIPv6Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
+	return rules
+}
+
+func defaultRule(rule *netlink.Rule) *netlink.Rule {
+	base := netlink.NewRule()
+	base.Src = rule.Src
+	base.Dst = rule.Dst
+	base.IifName = rule.IifName
+	base.Table = rule.Table
+	return base
+}

pkg/mesh/routes_test.go

@@ -0,0 +1,851 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mesh
+
+import (
+	"testing"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/squat/kilo/pkg/encapsulation"
+)
+
+func TestRoutes(t *testing.T) {
+	nodes, peers, key, port := setup(t)
+	kiloIface := 0
+	privIface := 1
+	tunlIface := 2
+	mustTopoForGranularityAndHost := func(granularity Granularity, hostname string) *Topology {
+		return mustTopo(t, nodes, peers, granularity, hostname, port, key, DefaultKiloSubnet, 0)
+	}
+
+	for _, tc := range []struct {
+		name     string
+		local    bool
+		topology *Topology
+		strategy encapsulation.Strategy
+		routes   []*netlink.Route
+		rules    []*netlink.Rule
+	}{
+		{
+			name:     "logical from a",
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].cidrs[1],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "logical from b",
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "logical from c",
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].wireGuardIP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[1].wireGuardIP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "full from a",
+			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "full from b",
+			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "full from c",
+			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].cidrs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "logical from a local",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       nodes["b"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["c"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "logical from a local always",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name),
+			strategy: encapsulation.Always,
+			routes: []*netlink.Route{
+				{
+					Dst:       nodes["b"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["c"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "logical from b local",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       nodes["a"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["c"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["c"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "logical from b local always",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name),
+			strategy: encapsulation.Always,
+			routes: []*netlink.Route{
+				{
+					Dst:       nodes["a"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["c"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["c"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["c"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+					Table:     kiloTableIndex,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+			rules: []*netlink.Rule{
+				defaultRule(&netlink.Rule{
+					Src:   nodes["b"].Subnet,
+					Dst:   nodes["c"].InternalIP,
+					Table: kiloTableIndex,
+				}),
+				defaultRule(&netlink.Rule{
+					Dst:     nodes["c"].InternalIP,
+					IifName: DefaultKiloInterface,
+					Table:   kiloTableIndex,
+				}),
+			},
+		},
+		{
+			name:     "logical from c local",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].wireGuardIP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["a"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[1].wireGuardIP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["b"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "logical from c local always",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name),
+			strategy: encapsulation.Always,
+			routes: []*netlink.Route{
+				{
+					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].wireGuardIP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["a"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[1].wireGuardIP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["b"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["b"].InternalIP,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+					Table:     kiloTableIndex,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        nodes["b"].InternalIP.IP,
+					LinkIndex: tunlIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+			rules: []*netlink.Rule{
+				defaultRule(&netlink.Rule{
+					Src:   nodes["c"].Subnet,
+					Dst:   nodes["b"].InternalIP,
+					Table: kiloTableIndex,
+				}),
+			},
+		},
+		{
+			name:     "full from a local",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       nodes["b"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["c"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "full from b local",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       nodes["a"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["c"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+		{
+			name:     "full from c local",
+			local:    true,
+			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name),
+			strategy: encapsulation.Never,
+			routes: []*netlink.Route{
+				{
+					Dst:       nodes["a"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       nodes["b"].Subnet,
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       peers["b"].AllowedIPs[0],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+			},
+		},
+	} {
+		routes, rules := tc.topology.Routes(DefaultKiloInterface, kiloIface, privIface, tunlIface, tc.local, encapsulation.NewIPIP(tc.strategy))
+		if diff := pretty.Compare(routes, tc.routes); diff != "" {
+			t.Errorf("test case %q: got diff: %v", tc.name, diff)
+		}
+		if diff := pretty.Compare(rules, tc.rules); diff != "" {
+			t.Errorf("test case %q: got diff: %v", tc.name, diff)
+		}
+	}
+}

pkg/mesh/topology.go

@@ -19,16 +19,9 @@ import (
 	"net"
 	"sort"
 
-	"github.com/vishvananda/netlink"
-	"golang.org/x/sys/unix"
-
-	"github.com/squat/kilo/pkg/encapsulation"
-	"github.com/squat/kilo/pkg/iptables"
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
-const kiloTableIndex = 1107
-
 // Topology represents the logical structure of the overlay network.
 type Topology struct {
 	// key is the private key of the node creating the topology.
@@ -165,193 +158,6 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 	return &t, nil
 }
 
-// Routes generates a slice of routes for a given Topology.
-func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface int, local bool, enc encapsulation.Encapsulator) ([]*netlink.Route, []*netlink.Rule) {
-	var routes []*netlink.Route
-	var rules []*netlink.Rule
-	if !t.leader {
-		// Find the GW for this segment.
-		// This will be the an IP of the leader.
-		// In an IPIP encapsulated mesh it is the leader's private IP.
-		var gw net.IP
-		for _, segment := range t.segments {
-			if segment.location == t.location {
-				gw = enc.Gw(segment.endpoint.IP, segment.privateIPs[segment.leader], segment.cidrs[segment.leader])
-				break
-			}
-		}
-		for _, segment := range t.segments {
-			// First, add a route to the WireGuard IP of the segment.
-			routes = append(routes, encapsulateRoute(&netlink.Route{
-				Dst:       oneAddressCIDR(segment.wireGuardIP),
-				Flags:     int(netlink.FLAG_ONLINK),
-				Gw:        gw,
-				LinkIndex: privIface,
-				Protocol:  unix.RTPROT_STATIC,
-			}, enc.Strategy(), t.privateIP, tunlIface))
-			// Add routes for the current segment if local is true.
-			if segment.location == t.location {
-				if local {
-					for i := range segment.cidrs {
-						// Don't add routes for the local node.
-						if segment.privateIPs[i].Equal(t.privateIP.IP) {
-							continue
-						}
-						routes = append(routes, encapsulateRoute(&netlink.Route{
-							Dst:       segment.cidrs[i],
-							Flags:     int(netlink.FLAG_ONLINK),
-							Gw:        segment.privateIPs[i],
-							LinkIndex: privIface,
-							Protocol:  unix.RTPROT_STATIC,
-						}, enc.Strategy(), t.privateIP, tunlIface))
-						// Encapsulate packets from the host's Pod subnet headed
-						// to private IPs.
-						if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
-							routes = append(routes, &netlink.Route{
-								Dst:       oneAddressCIDR(segment.privateIPs[i]),
-								Flags:     int(netlink.FLAG_ONLINK),
-								Gw:        segment.privateIPs[i],
-								LinkIndex: tunlIface,
-								Protocol:  unix.RTPROT_STATIC,
-								Table:     kiloTableIndex,
-							})
-							rules = append(rules, defaultRule(&netlink.Rule{
-								Src:   t.subnet,
-								Dst:   oneAddressCIDR(segment.privateIPs[i]),
-								Table: kiloTableIndex,
-							}))
-						}
-					}
-				}
-				continue
-			}
-			for i := range segment.cidrs {
-				// Add routes to the Pod CIDRs of nodes in other segments.
-				routes = append(routes, encapsulateRoute(&netlink.Route{
-					Dst:       segment.cidrs[i],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        gw,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				}, enc.Strategy(), t.privateIP, tunlIface))
-				// Add routes to the private IPs of nodes in other segments.
-				// Number of CIDRs and private IPs always match so
-				// we can reuse the loop.
-				routes = append(routes, encapsulateRoute(&netlink.Route{
-					Dst:       oneAddressCIDR(segment.privateIPs[i]),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        gw,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				}, enc.Strategy(), t.privateIP, tunlIface))
-			}
-		}
-		// Add routes for the allowed IPs of peers.
-		for _, peer := range t.peers {
-			for i := range peer.AllowedIPs {
-				routes = append(routes, encapsulateRoute(&netlink.Route{
-					Dst:       peer.AllowedIPs[i],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        gw,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				}, enc.Strategy(), t.privateIP, tunlIface))
-			}
-		}
-		return routes, rules
-	}
-	for _, segment := range t.segments {
-		// Add routes for the current segment if local is true.
-		if segment.location == t.location {
-			if local {
-				for i := range segment.cidrs {
-					// Don't add routes for the local node.
-					if segment.privateIPs[i].Equal(t.privateIP.IP) {
-						continue
-					}
-					routes = append(routes, encapsulateRoute(&netlink.Route{
-						Dst:       segment.cidrs[i],
-						Flags:     int(netlink.FLAG_ONLINK),
-						Gw:        segment.privateIPs[i],
-						LinkIndex: privIface,
-						Protocol:  unix.RTPROT_STATIC,
-					}, enc.Strategy(), t.privateIP, tunlIface))
-					// Encapsulate packets from the host's Pod subnet headed
-					// to private IPs.
-					if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
-						routes = append(routes, &netlink.Route{
-							Dst:       oneAddressCIDR(segment.privateIPs[i]),
-							Flags:     int(netlink.FLAG_ONLINK),
-							Gw:        segment.privateIPs[i],
-							LinkIndex: tunlIface,
-							Protocol:  unix.RTPROT_STATIC,
-							Table:     kiloTableIndex,
-						})
-						rules = append(rules, defaultRule(&netlink.Rule{
-							Src:   t.subnet,
-							Dst:   oneAddressCIDR(segment.privateIPs[i]),
-							Table: kiloTableIndex,
-						}))
-						// Also encapsulate packets from the Kilo interface
-						// headed to private IPs.
-						rules = append(rules, defaultRule(&netlink.Rule{
-							Dst:     oneAddressCIDR(segment.privateIPs[i]),
-							Table:   kiloTableIndex,
-							IifName: kiloIfaceName,
-						}))
-					}
-				}
-			}
-			continue
-		}
-		for i := range segment.cidrs {
-			// Add routes to the Pod CIDRs of nodes in other segments.
-			routes = append(routes, &netlink.Route{
-				Dst:       segment.cidrs[i],
-				Flags:     int(netlink.FLAG_ONLINK),
-				Gw:        segment.wireGuardIP,
-				LinkIndex: kiloIface,
-				Protocol:  unix.RTPROT_STATIC,
-			})
-			// Don't add routes through Kilo if the private IP
-			// equals the external IP. This means that the node
-			// is only accessible through an external IP and we
-			// cannot encapsulate traffic to an IP through the IP.
-			if segment.privateIPs[i].Equal(segment.endpoint.IP) {
-				continue
-			}
-			// Add routes to the private IPs of nodes in other segments.
-			// Number of CIDRs and private IPs always match so
-			// we can reuse the loop.
-			routes = append(routes, &netlink.Route{
-				Dst:       oneAddressCIDR(segment.privateIPs[i]),
-				Flags:     int(netlink.FLAG_ONLINK),
-				Gw:        segment.wireGuardIP,
-				LinkIndex: kiloIface,
-				Protocol:  unix.RTPROT_STATIC,
-			})
-		}
-	}
-	// Add routes for the allowed IPs of peers.
-	for _, peer := range t.peers {
-		for i := range peer.AllowedIPs {
-			routes = append(routes, &netlink.Route{
-				Dst:       peer.AllowedIPs[i],
-				LinkIndex: kiloIface,
-				Protocol:  unix.RTPROT_STATIC,
-			})
-		}
-	}
-	return routes, rules
-}
-
-func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route {
-	if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) {
-		route.LinkIndex = tunlIface
-	}
-	return route
-}
-
 // Conf generates a WireGuard configuration file for a given Topology.
 func (t *Topology) Conf() *wireguard.Conf {
 	c := &wireguard.Conf{
@@ -438,33 +244,6 @@ func (t *Topology) PeerConf(name string) *wireguard.Conf {
 	return c
 }
 
-// Rules returns the iptables rules required by the local node.
-func (t *Topology) Rules(cni bool) []iptables.Rule {
-	var rules []iptables.Rule
-	rules = append(rules, iptables.NewIPv4Chain("nat", "KILO-NAT"))
-	rules = append(rules, iptables.NewIPv6Chain("nat", "KILO-NAT"))
-	if cni {
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(t.subnet.IP)), "nat", "POSTROUTING", "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-s", t.subnet.String(), "-j", "KILO-NAT"))
-	}
-	for _, s := range t.segments {
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(s.wireGuardIP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuared IPs", "-d", s.wireGuardIP.String(), "-j", "RETURN"))
-		for _, aip := range s.allowedIPs {
-			rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-d", aip.String(), "-j", "RETURN"))
-		}
-	}
-	for _, p := range t.peers {
-		for _, aip := range p.AllowedIPs {
-			rules = append(rules,
-				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "POSTROUTING", "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-s", aip.String(), "-j", "KILO-NAT"),
-				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for peers", "-d", aip.String(), "-j", "RETURN"),
-			)
-		}
-	}
-	rules = append(rules, iptables.NewIPv4Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
-	rules = append(rules, iptables.NewIPv6Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
-	return rules
-}
-
 // oneAddressCIDR takes an IP address and returns a CIDR
 // that contains only that address.
 func oneAddressCIDR(ip net.IP) *net.IPNet {
@@ -522,12 +301,3 @@ func deduplicatePeerIPs(peers []*Peer) []*Peer {
 	}
 	return ps
 }
-
-func defaultRule(rule *netlink.Rule) *netlink.Rule {
-	base := netlink.NewRule()
-	base.Src = rule.Src
-	base.Dst = rule.Dst
-	base.IifName = rule.IifName
-	base.Table = rule.Table
-	return base
-}

pkg/mesh/topology_test.go

@@ -20,10 +20,8 @@ import (
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/squat/kilo/pkg/encapsulation"
+
 	"github.com/squat/kilo/pkg/wireguard"
-	"github.com/vishvananda/netlink"
-	"golang.org/x/sys/unix"
 )
 
 func allowedIPs(ips ...string) string {
@@ -372,832 +370,6 @@ func mustTopo(t *testing.T, nodes map[string]*Node, peers map[string]*Peer, gran
 	return topo
 }
 
-func TestRoutes(t *testing.T) {
-	nodes, peers, key, port := setup(t)
-	kiloIface := 0
-	privIface := 1
-	tunlIface := 2
-	mustTopoForGranularityAndHost := func(granularity Granularity, hostname string) *Topology {
-		return mustTopo(t, nodes, peers, granularity, hostname, port, key, DefaultKiloSubnet, 0)
-	}
-
-	for _, tc := range []struct {
-		name     string
-		local    bool
-		topology *Topology
-		strategy encapsulation.Strategy
-		routes   []*netlink.Route
-		rules    []*netlink.Rule
-	}{
-		{
-			name:     "logical from a",
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].cidrs[1],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "logical from b",
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "logical from c",
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].wireGuardIP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[1].wireGuardIP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "full from a",
-			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "full from b",
-			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "full from c",
-			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].cidrs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "logical from a local",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       nodes["b"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["c"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "logical from a local always",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name),
-			strategy: encapsulation.Always,
-			routes: []*netlink.Route{
-				{
-					Dst:       nodes["b"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["c"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "logical from b local",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       nodes["a"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["c"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["c"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "logical from b local always",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name),
-			strategy: encapsulation.Always,
-			routes: []*netlink.Route{
-				{
-					Dst:       nodes["a"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["c"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["c"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["c"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-					Table:     kiloTableIndex,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-			rules: []*netlink.Rule{
-				defaultRule(&netlink.Rule{
-					Src:   nodes["b"].Subnet,
-					Dst:   nodes["c"].InternalIP,
-					Table: kiloTableIndex,
-				}),
-				defaultRule(&netlink.Rule{
-					Dst:     nodes["c"].InternalIP,
-					IifName: DefaultKiloInterface,
-					Table:   kiloTableIndex,
-				}),
-			},
-		},
-		{
-			name:     "logical from c local",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].wireGuardIP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["a"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[1].wireGuardIP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["b"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "logical from c local always",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name),
-			strategy: encapsulation.Always,
-			routes: []*netlink.Route{
-				{
-					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[0].wireGuardIP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["a"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[1].wireGuardIP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["b"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["b"].InternalIP,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-					Table:     kiloTableIndex,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        nodes["b"].InternalIP.IP,
-					LinkIndex: tunlIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-			rules: []*netlink.Rule{
-				defaultRule(&netlink.Rule{
-					Src:   nodes["c"].Subnet,
-					Dst:   nodes["b"].InternalIP,
-					Table: kiloTableIndex,
-				}),
-			},
-		},
-		{
-			name:     "full from a local",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       nodes["b"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["c"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "full from b local",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       nodes["a"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["c"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["c"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-		{
-			name:     "full from c local",
-			local:    true,
-			topology: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name),
-			strategy: encapsulation.Never,
-			routes: []*netlink.Route{
-				{
-					Dst:       nodes["a"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["a"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       nodes["b"].Subnet,
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       oneAddressCIDR(nodes["b"].InternalIP.IP),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["a"].AllowedIPs[1],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-				{
-					Dst:       peers["b"].AllowedIPs[0],
-					LinkIndex: kiloIface,
-					Protocol:  unix.RTPROT_STATIC,
-				},
-			},
-		},
-	} {
-		routes, rules := tc.topology.Routes(DefaultKiloInterface, kiloIface, privIface, tunlIface, tc.local, encapsulation.NewIPIP(tc.strategy))
-		if diff := pretty.Compare(routes, tc.routes); diff != "" {
-			t.Errorf("test case %q: got diff: %v", tc.name, diff)
-		}
-		if diff := pretty.Compare(rules, tc.rules); diff != "" {
-			t.Errorf("test case %q: got diff: %v", tc.name, diff)
-		}
-	}
-}
-
 func TestConf(t *testing.T) {
 	nodes, peers, key, port := setup(t)
 	for _, tc := range []struct {

pkg/wireguard/wireguard.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build linux
+
 package wireguard
 
 import (