Add cross mesh granularity

cmd/kg/main.go

@@ -65,6 +65,7 @@ var (
 	availableGranularities = strings.Join([]string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.CrossGranularity),
 	}, ", ")
 	availableLogLevels = strings.Join([]string{
 		logLevelAll,
@@ -157,6 +158,7 @@ func Main() error {
 	switch gr {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.CrossGranularity:
 	default:
 		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", *granularity, availableGranularities)
 	}

docs/topology.md

@@ -46,6 +46,12 @@ kgctl graph | circo -Tsvg > cluster.svg
 
 <img src="./graphs/full-mesh.svg" />
 
+# Cross Mesh
+
+In this topology all nodes within the same location are not encrypted. Traffic to any other node outside of current location is encrypted
+with direct node-to-node encryption. To use this mesh specify `--mesh-granularity=cross`.
+
+
 ## Mixed 
 
 The `kilo.squat.ai/location` annotation can be used to create cluster mixing some fully meshed nodes and some nodes grouped by logical location.

pkg/mesh/backend.go

@@ -47,6 +47,9 @@ const (
 	// FullGranularity indicates that the network should create
 	// a mesh between every node.
 	FullGranularity Granularity = "full"
+	// CrossGranularity indicates that network is encrypted only
+	// between nodes in different locations.
+	CrossGranularity Granularity = "cross"
 )
 
 // Node represents a node in the network.

pkg/mesh/routes.go

@@ -125,7 +125,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 	}
 	for _, segment := range t.segments {
 		// Add routes for the current segment if local is true.
-		if segment.location == t.location {
+		if (segment.location == t.location) || (t.nodeLocation != "" && segment.nodeLocation == t.nodeLocation) {
 			if local {
 				for i := range segment.cidrs {
 					// Don't add routes for the local node.

pkg/mesh/topology.go

@@ -27,8 +27,11 @@ type Topology struct {
 	// key is the private key of the node creating the topology.
 	key  []byte
 	port uint32
-	// Location is the logical location of the local host.
+	// Location is the logical location of the local host of host name.
 	location string
+	// nodeLocation is the location annotation of the node. This is set only in cross location topology.
+	nodeLocation string
+
 	segments []*segment
 	peers    []*Peer
 
@@ -56,7 +59,8 @@ type segment struct {
 	key        []byte
 	// Location is the logical location of this segment.
 	location string
-
+	// nodeLocation is the node location annotation. This is set only for cross location topology.
+	nodeLocation string
 	// cidrs is a slice of subnets of all peers in the segment.
 	cidrs []*net.IPNet
 	// hostnames is a slice of the hostnames of the peers in the segment.
@@ -70,35 +74,58 @@ type segment struct {
 	wireGuardIP net.IP
 }
 
+// topoKey is used to group nodes into locations.
+type topoKey struct {
+	location     string
+	nodeLocation string
+}
+
 // NewTopology creates a new Topology struct from a given set of nodes and peers.
 func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port uint32, key []byte, subnet *net.IPNet, persistentKeepalive int) (*Topology, error) {
-	topoMap := make(map[string][]*Node)
+	topoMap := make(map[topoKey][]*Node)
-	for _, node := range nodes {
+	var localLocation, localNodeLocation string
-		var location string
-		switch granularity {
-		case LogicalGranularity:
-			location = node.Location
-		case FullGranularity:
-			location = node.Name
-		}
-		topoMap[location] = append(topoMap[location], node)
-	}
-	var localLocation string
 	switch granularity {
 	case LogicalGranularity:
 		localLocation = nodes[hostname].Location
+	case CrossGranularity:
+		localLocation = hostname
+		localNodeLocation = nodes[hostname].Location
 	case FullGranularity:
 		localLocation = hostname
 	}
 
-	t := Topology{key: key, port: port, hostname: hostname, location: localLocation, persistentKeepalive: persistentKeepalive, privateIP: nodes[hostname].InternalIP, subnet: nodes[hostname].Subnet}
+	for _, node := range nodes {
+		var location, nodeLocation string
+		switch granularity {
+		case LogicalGranularity:
+			location = node.Location
+		case CrossGranularity:
+			location = node.Name
+			nodeLocation = node.Location
+		case FullGranularity:
+			location = node.Name
+		}
+		key := topoKey{location: location, nodeLocation: nodeLocation}
+		topoMap[key] = append(topoMap[key], node)
+	}
+
+	t := Topology{
+		key:                 key,
+		port:                port,
+		hostname:            hostname,
+		location:            localLocation,
+		nodeLocation:        localNodeLocation,
+		persistentKeepalive: persistentKeepalive,
+		privateIP:           nodes[hostname].InternalIP,
+		subnet:              nodes[hostname].Subnet,
+	}
 	for location := range topoMap {
 		// Sort the location so the result is stable.
 		sort.Slice(topoMap[location], func(i, j int) bool {
 			return topoMap[location][i].Name < topoMap[location][j].Name
 		})
 		leader := findLeader(topoMap[location])
-		if location == localLocation && topoMap[location][leader].Name == hostname {
+		if location.nodeLocation != "" || (location.location == localLocation && topoMap[location][leader].Name == hostname) {
 			t.leader = true
 		}
 		var allowedIPs []*net.IPNet
@@ -116,14 +143,15 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			privateIPs = append(privateIPs, node.InternalIP.IP)
 		}
 		t.segments = append(t.segments, &segment{
-			allowedIPs: allowedIPs,
+			allowedIPs:   allowedIPs,
-			endpoint:   topoMap[location][leader].Endpoint,
+			endpoint:     topoMap[location][leader].Endpoint,
-			key:        topoMap[location][leader].Key,
+			key:          topoMap[location][leader].Key,
-			location:   location,
+			location:     location.location,
-			cidrs:      cidrs,
+			nodeLocation: location.nodeLocation,
-			hostnames:  hostnames,
+			cidrs:        cidrs,
-			leader:     leader,
+			hostnames:    hostnames,
-			privateIPs: privateIPs,
+			leader:       leader,
+			privateIPs:   privateIPs,
 		})
 	}
 	// Sort the Topology segments so the result is stable.
@@ -167,7 +195,7 @@ func (t *Topology) Conf() *wireguard.Conf {
 		},
 	}
 	for _, s := range t.segments {
-		if s.location == t.location {
+		if (s.location == t.location) || (t.nodeLocation != "" && t.nodeLocation == s.nodeLocation) {
 			continue
 		}
 		peer := &wireguard.Peer{

pkg/mesh/topology_test.go

@@ -349,6 +349,156 @@ func TestNewTopology(t *testing.T) {
 				peers: []*Peer{peers["a"], peers["b"]},
 			},
 		},
+		{
+			name:        "cross from a",
+			granularity: CrossGranularity,
+			hostname:    nodes["a"].Name,
+			result: &Topology{
+				hostname:      nodes["a"].Name,
+				leader:        true,
+				location:      nodes["a"].Name,
+				nodeLocation:  nodes["a"].Location,
+				subnet:        nodes["a"].Subnet,
+				privateIP:     nodes["a"].InternalIP,
+				wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)},
+				segments: []*segment{
+					{
+						allowedIPs:   []*net.IPNet{nodes["a"].Subnet, nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["a"].Endpoint,
+						key:          nodes["a"].Key,
+						location:     nodes["a"].Name,
+						nodeLocation: nodes["a"].Location,
+						cidrs:        []*net.IPNet{nodes["a"].Subnet},
+						hostnames:    []string{"a"},
+						privateIPs:   []net.IP{nodes["a"].InternalIP.IP},
+						wireGuardIP:  w1,
+					},
+					{
+						allowedIPs:   []*net.IPNet{nodes["b"].Subnet, nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["b"].Endpoint,
+						key:          nodes["b"].Key,
+						location:     nodes["b"].Name,
+						nodeLocation: nodes["b"].Location,
+						cidrs:        []*net.IPNet{nodes["b"].Subnet},
+						hostnames:    []string{"b"},
+						privateIPs:   []net.IP{nodes["b"].InternalIP.IP},
+						wireGuardIP:  w2,
+					},
+					{
+						allowedIPs:   []*net.IPNet{nodes["c"].Subnet, nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["c"].Endpoint,
+						key:          nodes["c"].Key,
+						location:     nodes["c"].Name,
+						nodeLocation: nodes["c"].Location,
+						cidrs:        []*net.IPNet{nodes["c"].Subnet},
+						hostnames:    []string{"c"},
+						privateIPs:   []net.IP{nodes["c"].InternalIP.IP},
+						wireGuardIP:  w3,
+					},
+				},
+				peers: []*Peer{peers["a"], peers["b"]},
+			},
+		},
+		{
+			name:        "cross from b",
+			granularity: CrossGranularity,
+			hostname:    nodes["b"].Name,
+			result: &Topology{
+				hostname:      nodes["b"].Name,
+				leader:        true,
+				location:      nodes["b"].Name,
+				nodeLocation:  nodes["b"].Location,
+				subnet:        nodes["b"].Subnet,
+				privateIP:     nodes["b"].InternalIP,
+				wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)},
+				segments: []*segment{
+					{
+						allowedIPs:   []*net.IPNet{nodes["a"].Subnet, nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["a"].Endpoint,
+						key:          nodes["a"].Key,
+						location:     nodes["a"].Name,
+						nodeLocation: nodes["a"].Location,
+						cidrs:        []*net.IPNet{nodes["a"].Subnet},
+						hostnames:    []string{"a"},
+						privateIPs:   []net.IP{nodes["a"].InternalIP.IP},
+						wireGuardIP:  w1,
+					},
+					{
+						allowedIPs:   []*net.IPNet{nodes["b"].Subnet, nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["b"].Endpoint,
+						key:          nodes["b"].Key,
+						location:     nodes["b"].Name,
+						nodeLocation: nodes["b"].Location,
+						cidrs:        []*net.IPNet{nodes["b"].Subnet},
+						hostnames:    []string{"b"},
+						privateIPs:   []net.IP{nodes["b"].InternalIP.IP},
+						wireGuardIP:  w2,
+					},
+					{
+						allowedIPs:   []*net.IPNet{nodes["c"].Subnet, nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["c"].Endpoint,
+						key:          nodes["c"].Key,
+						location:     nodes["c"].Name,
+						nodeLocation: nodes["c"].Location,
+						cidrs:        []*net.IPNet{nodes["c"].Subnet},
+						hostnames:    []string{"c"},
+						privateIPs:   []net.IP{nodes["c"].InternalIP.IP},
+						wireGuardIP:  w3,
+					},
+				},
+				peers: []*Peer{peers["a"], peers["b"]},
+			},
+		},
+		{
+			name:        "cross from c",
+			granularity: CrossGranularity,
+			hostname:    nodes["c"].Name,
+			result: &Topology{
+				hostname:      nodes["c"].Name,
+				leader:        true,
+				location:      nodes["c"].Name,
+				nodeLocation:  nodes["c"].Location,
+				subnet:        nodes["c"].Subnet,
+				privateIP:     nodes["c"].InternalIP,
+				wireGuardCIDR: &net.IPNet{IP: w3, Mask: net.CIDRMask(16, 32)},
+				segments: []*segment{
+					{
+						allowedIPs:   []*net.IPNet{nodes["a"].Subnet, nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["a"].Endpoint,
+						key:          nodes["a"].Key,
+						location:     nodes["a"].Name,
+						nodeLocation: nodes["a"].Location,
+						cidrs:        []*net.IPNet{nodes["a"].Subnet},
+						hostnames:    []string{"a"},
+						privateIPs:   []net.IP{nodes["a"].InternalIP.IP},
+						wireGuardIP:  w1,
+					},
+					{
+						allowedIPs:   []*net.IPNet{nodes["b"].Subnet, nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["b"].Endpoint,
+						key:          nodes["b"].Key,
+						location:     nodes["b"].Name,
+						nodeLocation: nodes["b"].Location,
+						cidrs:        []*net.IPNet{nodes["b"].Subnet},
+						hostnames:    []string{"b"},
+						privateIPs:   []net.IP{nodes["b"].InternalIP.IP},
+						wireGuardIP:  w2,
+					},
+					{
+						allowedIPs:   []*net.IPNet{nodes["c"].Subnet, nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}},
+						endpoint:     nodes["c"].Endpoint,
+						key:          nodes["c"].Key,
+						location:     nodes["c"].Name,
+						nodeLocation: nodes["c"].Location,
+						cidrs:        []*net.IPNet{nodes["c"].Subnet},
+						hostnames:    []string{"c"},
+						privateIPs:   []net.IP{nodes["c"].InternalIP.IP},
+						wireGuardIP:  w3,
+					},
+				},
+				peers: []*Peer{peers["a"], peers["b"]},
+			},
+		},
 	} {
 		tc.result.key = key
 		tc.result.port = port
@@ -531,6 +681,85 @@ PersistentKeepalive = 25
 		AllowedIPs = 10.5.0.3/24
 		`,
 		},
+		{
+			name:     "cross from a",
+			topology: mustTopo(t, nodes, peers, CrossGranularity, nodes["a"].Name, port, key, DefaultKiloSubnet, nodes["a"].PersistentKeepalive),
+			result: `[Interface]
+		PrivateKey = private
+		ListenPort = 51820
+
+		[Peer]
+		AllowedIPs = 10.2.2.0/24, 192.168.0.1/32, 10.4.0.2/32
+		Endpoint = 10.1.0.2:51820
+		PersistentKeepalive = 25
+		PublicKey = key2
+
+		[Peer]
+		AllowedIPs = 10.2.3.0/24, 192.168.0.2/32, 10.4.0.3/32
+		Endpoint = 10.1.0.3:51820
+		PersistentKeepalive = 25
+		PublicKey = key3
+
+		[Peer] 
+		AllowedIPs = 10.5.0.1/24, 10.5.0.2/24
+		PersistentKeepalive = 25
+		PublicKey = key4
+
+		[Peer]
+		PublicKey = key5
+		Endpoint = 192.168.0.1:51820
+		PersistentKeepalive = 25
+		AllowedIPs = 10.5.0.3/24
+		`,
+		},
+		{
+			name:     "cross from b",
+			topology: mustTopo(t, nodes, peers, CrossGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet, nodes["b"].PersistentKeepalive),
+			result: `[Interface]
+		PrivateKey = private
+		ListenPort = 51820
+		
+		[Peer]
+		PublicKey = key1
+		Endpoint = 10.1.0.1:51820
+		AllowedIPs = 10.2.1.0/24, 192.168.0.1/32, 10.4.0.1/32
+
+        [Peer]
+        AllowedIPs = 10.5.0.1/24, 10.5.0.2/24
+        PersistentKeepalive = 0
+        PublicKey = key4
+        
+        [Peer]
+        AllowedIPs = 10.5.0.3/24
+        Endpoint = [192.168.0.1]:51820
+        PersistentKeepalive = 0
+        PublicKey = key5
+		`,
+		},
+		{
+			name:     "cross from c",
+			topology: mustTopo(t, nodes, peers, CrossGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet, nodes["b"].PersistentKeepalive),
+			result: `[Interface]
+		PrivateKey = private
+		ListenPort = 51820
+		
+		[Peer]
+		PublicKey = key1
+		Endpoint = 10.1.0.1:51820
+		AllowedIPs = 10.2.1.0/24, 192.168.0.1/32, 10.4.0.1/32
+
+        [Peer]
+        AllowedIPs = 10.5.0.1/24, 10.5.0.2/24
+        PersistentKeepalive = 0
+        PublicKey = key4
+        
+        [Peer]
+        AllowedIPs = 10.5.0.3/24
+        Endpoint = [192.168.0.1]:51820
+        PersistentKeepalive = 0
+        PublicKey = key5
+		`,
+		},
 	} {
 		conf := tc.topology.Conf()
 		if !conf.Equal(wireguard.Parse([]byte(tc.result))) {