Add cross mesh granularity

cmd/kg/main.go

@@ -69,6 +69,7 @@ var (
 	availableGranularities = strings.Join([]string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.CrossGranularity),
 	}, ", ")
 	availableLogLevels = strings.Join([]string{
 		logLevelAll,
@@ -223,6 +224,7 @@ func runRoot(_ *cobra.Command, _ []string) error {
 	switch gr {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.CrossGranularity:
 	default:
 		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", granularity, availableGranularities)
 	}

docs/topology.md

@@ -47,6 +47,11 @@ kgctl graph | circo -Tsvg > cluster.svg
 
 <img src="./graphs/full-mesh.svg" />
 
+# Cross Mesh
+
+In this topology all nodes within the same location are not encrypted. Traffic to any other node outside of current location is encrypted
+with direct node-to-node encryption. To use this mesh specify `--mesh-granularity=cross`.
+
 ## Mixed 
 
 The `kilo.squat.ai/location` annotation can be used to create cluster mixing some fully meshed nodes and some nodes grouped by logical location.

pkg/mesh/backend.go

@@ -50,6 +50,9 @@ const (
 	// FullGranularity indicates that the network should create
 	// a mesh between every node.
 	FullGranularity Granularity = "full"
+	// CrossGranularity indicates that network is encrypted only
+	// between nodes in different locations.
+	CrossGranularity Granularity = "cross"
 	// AutoGranularity can be used with kgctl to obtain
 	// the granularity automatically.
 	AutoGranularity Granularity = "auto"

pkg/mesh/routes.go

@@ -137,7 +137,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 	}
 	for _, segment := range t.segments {
 		// Add routes for the current segment if local is true.
-		if segment.location == t.location {
+		if (segment.location == t.location) || (t.nodeLocation != "" && segment.nodeLocation == t.nodeLocation) {
 			// If the local node does not have a private IP address,
 			// then skip adding routes, because the node is in its own location.
 			if local && t.privateIP != nil {

pkg/mesh/topology.go

@@ -37,10 +37,12 @@ type Topology struct {
 	// key is the private key of the node creating the topology.
 	key  wgtypes.Key
 	port int
-	// Location is the logical location of the local host.
+	// location is the logical location of the local host.
 	location string
-	segments []*segment
+	// nodeLocation is the location annotation of the node. This is set only in cross location topology.
-	peers    []*Peer
+	nodeLocation string
+	segments     []*segment
+	peers        []*Peer
 
 	// hostname is the hostname of the local host.
 	hostname string
@@ -71,8 +73,10 @@ type segment struct {
 	endpoint            *wireguard.Endpoint
 	key                 wgtypes.Key
 	persistentKeepalive time.Duration
-	// Location is the logical location of this segment.
+	// location is the logical location of this segment.
 	location string
+	// nodeLocation is the node location annotation. This is set only for cross location topology.
+	nodeLocation string
 
 	// cidrs is a slice of subnets of all peers in the segment.
 	cidrs []*net.IPNet
@@ -91,14 +95,34 @@ type segment struct {
 	allowedLocationIPs []net.IPNet
 }
 
+// topoKey is used to group nodes into locations.
+type topoKey struct {
+	location     string
+	nodeLocation string
+}
+
 // NewTopology creates a new Topology struct from a given set of nodes and peers.
 func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port int, key wgtypes.Key, subnet *net.IPNet, persistentKeepalive time.Duration, logger log.Logger) (*Topology, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
-	topoMap := make(map[string][]*Node)
+	topoMap := make(map[topoKey][]*Node)
+	var localLocation, localNodeLocation string
+	switch granularity {
+	case LogicalGranularity:
+		localLocation = logicalLocationPrefix + nodes[hostname].Location
+		if nodes[hostname].InternalIP == nil {
+			localLocation = nodeLocationPrefix + hostname
+		}
+	case FullGranularity:
+		localLocation = nodeLocationPrefix + hostname
+	case CrossGranularity:
+		localLocation = nodeLocationPrefix + hostname
+		localNodeLocation = logicalLocationPrefix + nodes[hostname].Location
+	}
+
 	for _, node := range nodes {
-		var location string
+		var location, nodeLocation string
 		switch granularity {
 		case LogicalGranularity:
 			location = logicalLocationPrefix + node.Location
@@ -109,18 +133,12 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			}
 		case FullGranularity:
 			location = nodeLocationPrefix + node.Name
+		case CrossGranularity:
+			location = nodeLocationPrefix + node.Name
+			nodeLocation = logicalLocationPrefix + node.Location
 		}
-		topoMap[location] = append(topoMap[location], node)
+		key := topoKey{location: location, nodeLocation: nodeLocation}
-	}
+		topoMap[key] = append(topoMap[key], node)
-	var localLocation string
-	switch granularity {
-	case LogicalGranularity:
-		localLocation = logicalLocationPrefix + nodes[hostname].Location
-		if nodes[hostname].InternalIP == nil {
-			localLocation = nodeLocationPrefix + hostname
-		}
-	case FullGranularity:
-		localLocation = nodeLocationPrefix + hostname
 	}
 
 	t := Topology{
@@ -128,6 +146,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 		port:                port,
 		hostname:            hostname,
 		location:            localLocation,
+		nodeLocation:        localNodeLocation,
 		persistentKeepalive: persistentKeepalive,
 		privateIP:           nodes[hostname].InternalIP,
 		subnet:              nodes[hostname].Subnet,
@@ -141,7 +160,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			return topoMap[location][i].Name < topoMap[location][j].Name
 		})
 		leader := findLeader(topoMap[location])
-		if location == localLocation && topoMap[location][leader].Name == hostname {
+		if location.nodeLocation != "" || (location.location == localLocation && topoMap[location][leader].Name == hostname) {
 			t.leader = true
 		}
 		var allowedIPs []net.IPNet
@@ -181,7 +200,8 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			endpoint:            topoMap[location][leader].Endpoint,
 			key:                 topoMap[location][leader].Key,
 			persistentKeepalive: topoMap[location][leader].PersistentKeepalive,
-			location:            location,
+			location:            location.location,
+			nodeLocation:        location.nodeLocation,
 			cidrs:               cidrs,
 			hostnames:           hostnames,
 			leader:              leader,
@@ -225,7 +245,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 
 		// Now that the topology is ordered, update the discoveredEndpoints map
 		// add new ones by going through the ordered topology: segments, nodes
-		for _, node := range topoMap[segment.location] {
+		for _, node := range topoMap[topoKey{location: segment.location, nodeLocation: segment.nodeLocation}] {
 			for key := range node.DiscoveredEndpoints {
 				if _, ok := t.discoveredEndpoints[key]; !ok {
 					t.discoveredEndpoints[key] = node.DiscoveredEndpoints[key]
@@ -308,7 +328,7 @@ func (t *Topology) Conf() *wireguard.Conf {
 		},
 	}
 	for _, s := range t.segments {
-		if s.location == t.location {
+		if (s.location == t.location) || (t.nodeLocation != "" && t.nodeLocation == s.nodeLocation) {
 			continue
 		}
 		peer := wireguard.Peer{

pkg/mesh/topology_test.go

@@ -532,6 +532,274 @@ func TestNewTopology(t *testing.T) {
 				logger: log.NewNopLogger(),
 			},
 		},
+		{
+			name:        "cross from a",
+			granularity: CrossGranularity,
+			hostname:    nodes["a"].Name,
+			result: &Topology{
+				hostname:      nodes["a"].Name,
+				leader:        true,
+				location:      nodeLocationPrefix + nodes["a"].Name,
+				nodeLocation:  logicalLocationPrefix + nodes["a"].Location,
+				subnet:        nodes["a"].Subnet,
+				privateIP:     nodes["a"].InternalIP,
+				wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)},
+				segments: []*segment{
+					{
+						allowedIPs:          []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["a"].Endpoint,
+						key:                 nodes["a"].Key,
+						persistentKeepalive: nodes["a"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["a"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["a"].Location,
+						cidrs:               []*net.IPNet{nodes["a"].Subnet},
+						hostnames:           []string{"a"},
+						privateIPs:          []net.IP{nodes["a"].InternalIP.IP},
+						wireGuardIP:         w1,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["b"].Endpoint,
+						key:                 nodes["b"].Key,
+						persistentKeepalive: nodes["b"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["b"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["b"].Location,
+						cidrs:               []*net.IPNet{nodes["b"].Subnet},
+						hostnames:           []string{"b"},
+						privateIPs:          []net.IP{nodes["b"].InternalIP.IP},
+						wireGuardIP:         w2,
+						allowedLocationIPs:  nodes["b"].AllowedLocationIPs,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["c"].Endpoint,
+						key:                 nodes["c"].Key,
+						persistentKeepalive: nodes["c"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["c"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["c"].Location,
+						cidrs:               []*net.IPNet{nodes["c"].Subnet},
+						hostnames:           []string{"c"},
+						privateIPs:          []net.IP{nodes["c"].InternalIP.IP},
+						wireGuardIP:         w3,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["d"].Endpoint,
+						key:                 nodes["d"].Key,
+						persistentKeepalive: nodes["d"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["d"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["d"].Location,
+						cidrs:               []*net.IPNet{nodes["d"].Subnet},
+						hostnames:           []string{"d"},
+						privateIPs:          nil,
+						wireGuardIP:         w4,
+					},
+				},
+				peers:  []*Peer{peers["a"], peers["b"]},
+				logger: log.NewNopLogger(),
+			},
+		},
+		{
+			name:        "cross from b",
+			granularity: CrossGranularity,
+			hostname:    nodes["b"].Name,
+			result: &Topology{
+				hostname:      nodes["b"].Name,
+				leader:        true,
+				location:      nodeLocationPrefix + nodes["b"].Name,
+				nodeLocation:  logicalLocationPrefix + nodes["b"].Location,
+				subnet:        nodes["b"].Subnet,
+				privateIP:     nodes["b"].InternalIP,
+				wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)},
+				segments: []*segment{
+					{
+						allowedIPs:          []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["a"].Endpoint,
+						key:                 nodes["a"].Key,
+						persistentKeepalive: nodes["a"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["a"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["a"].Location,
+						cidrs:               []*net.IPNet{nodes["a"].Subnet},
+						hostnames:           []string{"a"},
+						privateIPs:          []net.IP{nodes["a"].InternalIP.IP},
+						wireGuardIP:         w1,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["b"].Endpoint,
+						key:                 nodes["b"].Key,
+						persistentKeepalive: nodes["b"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["b"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["b"].Location,
+						cidrs:               []*net.IPNet{nodes["b"].Subnet},
+						hostnames:           []string{"b"},
+						privateIPs:          []net.IP{nodes["b"].InternalIP.IP},
+						wireGuardIP:         w2,
+						allowedLocationIPs:  nodes["b"].AllowedLocationIPs,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["c"].Endpoint,
+						key:                 nodes["c"].Key,
+						persistentKeepalive: nodes["c"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["c"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["c"].Location,
+						cidrs:               []*net.IPNet{nodes["c"].Subnet},
+						hostnames:           []string{"c"},
+						privateIPs:          []net.IP{nodes["c"].InternalIP.IP},
+						wireGuardIP:         w3,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["d"].Endpoint,
+						key:                 nodes["d"].Key,
+						persistentKeepalive: nodes["d"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["d"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["d"].Location,
+						cidrs:               []*net.IPNet{nodes["d"].Subnet},
+						hostnames:           []string{"d"},
+						privateIPs:          nil,
+						wireGuardIP:         w4,
+					},
+				},
+				peers:  []*Peer{peers["a"], peers["b"]},
+				logger: log.NewNopLogger(),
+			},
+		},
+		{
+			name:        "cross from c",
+			granularity: CrossGranularity,
+			hostname:    nodes["c"].Name,
+			result: &Topology{
+				hostname:      nodes["c"].Name,
+				leader:        true,
+				location:      nodeLocationPrefix + nodes["c"].Name,
+				nodeLocation:  logicalLocationPrefix + nodes["c"].Location,
+				subnet:        nodes["c"].Subnet,
+				privateIP:     nodes["c"].InternalIP,
+				wireGuardCIDR: &net.IPNet{IP: w3, Mask: net.CIDRMask(16, 32)},
+				segments: []*segment{
+					{
+						allowedIPs:          []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["a"].Endpoint,
+						key:                 nodes["a"].Key,
+						persistentKeepalive: nodes["a"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["a"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["a"].Location,
+						cidrs:               []*net.IPNet{nodes["a"].Subnet},
+						hostnames:           []string{"a"},
+						privateIPs:          []net.IP{nodes["a"].InternalIP.IP},
+						wireGuardIP:         w1,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["b"].Endpoint,
+						key:                 nodes["b"].Key,
+						persistentKeepalive: nodes["b"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["b"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["b"].Location,
+						cidrs:               []*net.IPNet{nodes["b"].Subnet},
+						hostnames:           []string{"b"},
+						privateIPs:          []net.IP{nodes["b"].InternalIP.IP},
+						wireGuardIP:         w2,
+						allowedLocationIPs:  nodes["b"].AllowedLocationIPs,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["c"].Endpoint,
+						key:                 nodes["c"].Key,
+						persistentKeepalive: nodes["c"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["c"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["c"].Location,
+						cidrs:               []*net.IPNet{nodes["c"].Subnet},
+						hostnames:           []string{"c"},
+						privateIPs:          []net.IP{nodes["c"].InternalIP.IP},
+						wireGuardIP:         w3,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["d"].Endpoint,
+						key:                 nodes["d"].Key,
+						persistentKeepalive: nodes["d"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["d"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["d"].Location,
+						cidrs:               []*net.IPNet{nodes["d"].Subnet},
+						hostnames:           []string{"d"},
+						privateIPs:          nil,
+						wireGuardIP:         w4,
+					},
+				},
+				peers:  []*Peer{peers["a"], peers["b"]},
+				logger: log.NewNopLogger(),
+			},
+		},
+		{
+			name:        "cross from d",
+			granularity: CrossGranularity,
+			hostname:    nodes["d"].Name,
+			result: &Topology{
+				hostname:      nodes["d"].Name,
+				leader:        true,
+				location:      nodeLocationPrefix + nodes["d"].Name,
+				nodeLocation:  logicalLocationPrefix + nodes["d"].Location,
+				subnet:        nodes["d"].Subnet,
+				privateIP:     nil,
+				wireGuardCIDR: &net.IPNet{IP: w4, Mask: net.CIDRMask(16, 32)},
+				segments: []*segment{
+					{
+						allowedIPs:          []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["a"].Endpoint,
+						key:                 nodes["a"].Key,
+						persistentKeepalive: nodes["a"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["a"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["a"].Location,
+						cidrs:               []*net.IPNet{nodes["a"].Subnet},
+						hostnames:           []string{"a"},
+						privateIPs:          []net.IP{nodes["a"].InternalIP.IP},
+						wireGuardIP:         w1,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["b"].Endpoint,
+						key:                 nodes["b"].Key,
+						persistentKeepalive: nodes["b"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["b"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["b"].Location,
+						cidrs:               []*net.IPNet{nodes["b"].Subnet},
+						hostnames:           []string{"b"},
+						privateIPs:          []net.IP{nodes["b"].InternalIP.IP},
+						wireGuardIP:         w2,
+						allowedLocationIPs:  nodes["b"].AllowedLocationIPs,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["c"].Endpoint,
+						key:                 nodes["c"].Key,
+						persistentKeepalive: nodes["c"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["c"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["c"].Location,
+						cidrs:               []*net.IPNet{nodes["c"].Subnet},
+						hostnames:           []string{"c"},
+						privateIPs:          []net.IP{nodes["c"].InternalIP.IP},
+						wireGuardIP:         w3,
+					},
+					{
+						allowedIPs:          []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}},
+						endpoint:            nodes["d"].Endpoint,
+						key:                 nodes["d"].Key,
+						persistentKeepalive: nodes["d"].PersistentKeepalive,
+						location:            nodeLocationPrefix + nodes["d"].Name,
+						nodeLocation:        logicalLocationPrefix + nodes["d"].Location,
+						cidrs:               []*net.IPNet{nodes["d"].Subnet},
+						hostnames:           []string{"d"},
+						privateIPs:          nil,
+						wireGuardIP:         w4,
+					},
+				},
+				peers:  []*Peer{peers["a"], peers["b"]},
+				logger: log.NewNopLogger(),
+			},
+		},
 	} {
 		tc.result.key = key
 		tc.result.port = port