diff --git a/cmd/kg/main.go b/cmd/kg/main.go index 1834653..89f1448 100644 --- a/cmd/kg/main.go +++ b/cmd/kg/main.go @@ -69,6 +69,7 @@ var ( availableGranularities = strings.Join([]string{ string(mesh.LogicalGranularity), string(mesh.FullGranularity), + string(mesh.CrossGranularity), }, ", ") availableLogLevels = strings.Join([]string{ logLevelAll, @@ -223,6 +224,7 @@ func runRoot(_ *cobra.Command, _ []string) error { switch gr { case mesh.LogicalGranularity: case mesh.FullGranularity: + case mesh.CrossGranularity: default: return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", granularity, availableGranularities) } diff --git a/docs/topology.md b/docs/topology.md index e5eafd0..3fbef69 100644 --- a/docs/topology.md +++ b/docs/topology.md @@ -47,6 +47,11 @@ kgctl graph | circo -Tsvg > cluster.svg +# Cross Mesh + +In this topology all nodes within the same location are not encrypted. Traffic to any other node outside of current location is encrypted +with direct node-to-node encryption. To use this mesh specify `--mesh-granularity=cross`. + ## Mixed The `kilo.squat.ai/location` annotation can be used to create cluster mixing some fully meshed nodes and some nodes grouped by logical location. diff --git a/pkg/mesh/backend.go b/pkg/mesh/backend.go index 203661d..79e0319 100644 --- a/pkg/mesh/backend.go +++ b/pkg/mesh/backend.go @@ -50,6 +50,9 @@ const ( // FullGranularity indicates that the network should create // a mesh between every node. FullGranularity Granularity = "full" + // CrossGranularity indicates that network is encrypted only + // between nodes in different locations. + CrossGranularity Granularity = "cross" // AutoGranularity can be used with kgctl to obtain // the granularity automatically. AutoGranularity Granularity = "auto" diff --git a/pkg/mesh/routes.go b/pkg/mesh/routes.go index 2c0cf24..7889606 100644 --- a/pkg/mesh/routes.go +++ b/pkg/mesh/routes.go @@ -137,7 +137,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface } for _, segment := range t.segments { // Add routes for the current segment if local is true. - if segment.location == t.location { + if (segment.location == t.location) || (t.nodeLocation != "" && segment.nodeLocation == t.nodeLocation) { // If the local node does not have a private IP address, // then skip adding routes, because the node is in its own location. if local && t.privateIP != nil { diff --git a/pkg/mesh/topology.go b/pkg/mesh/topology.go index c7aaff7..bdb0610 100644 --- a/pkg/mesh/topology.go +++ b/pkg/mesh/topology.go @@ -37,10 +37,12 @@ type Topology struct { // key is the private key of the node creating the topology. key wgtypes.Key port int - // Location is the logical location of the local host. + // location is the logical location of the local host. location string - segments []*segment - peers []*Peer + // nodeLocation is the location annotation of the node. This is set only in cross location topology. + nodeLocation string + segments []*segment + peers []*Peer // hostname is the hostname of the local host. hostname string @@ -71,8 +73,10 @@ type segment struct { endpoint *wireguard.Endpoint key wgtypes.Key persistentKeepalive time.Duration - // Location is the logical location of this segment. + // location is the logical location of this segment. location string + // nodeLocation is the node location annotation. This is set only for cross location topology. + nodeLocation string // cidrs is a slice of subnets of all peers in the segment. cidrs []*net.IPNet @@ -91,14 +95,34 @@ type segment struct { allowedLocationIPs []net.IPNet } +// topoKey is used to group nodes into locations. +type topoKey struct { + location string + nodeLocation string +} + // NewTopology creates a new Topology struct from a given set of nodes and peers. func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port int, key wgtypes.Key, subnet *net.IPNet, persistentKeepalive time.Duration, logger log.Logger) (*Topology, error) { if logger == nil { logger = log.NewNopLogger() } - topoMap := make(map[string][]*Node) + topoMap := make(map[topoKey][]*Node) + var localLocation, localNodeLocation string + switch granularity { + case LogicalGranularity: + localLocation = logicalLocationPrefix + nodes[hostname].Location + if nodes[hostname].InternalIP == nil { + localLocation = nodeLocationPrefix + hostname + } + case FullGranularity: + localLocation = nodeLocationPrefix + hostname + case CrossGranularity: + localLocation = nodeLocationPrefix + hostname + localNodeLocation = logicalLocationPrefix + nodes[hostname].Location + } + for _, node := range nodes { - var location string + var location, nodeLocation string switch granularity { case LogicalGranularity: location = logicalLocationPrefix + node.Location @@ -109,18 +133,12 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra } case FullGranularity: location = nodeLocationPrefix + node.Name + case CrossGranularity: + location = nodeLocationPrefix + node.Name + nodeLocation = logicalLocationPrefix + node.Location } - topoMap[location] = append(topoMap[location], node) - } - var localLocation string - switch granularity { - case LogicalGranularity: - localLocation = logicalLocationPrefix + nodes[hostname].Location - if nodes[hostname].InternalIP == nil { - localLocation = nodeLocationPrefix + hostname - } - case FullGranularity: - localLocation = nodeLocationPrefix + hostname + key := topoKey{location: location, nodeLocation: nodeLocation} + topoMap[key] = append(topoMap[key], node) } t := Topology{ @@ -128,6 +146,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra port: port, hostname: hostname, location: localLocation, + nodeLocation: localNodeLocation, persistentKeepalive: persistentKeepalive, privateIP: nodes[hostname].InternalIP, subnet: nodes[hostname].Subnet, @@ -141,7 +160,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra return topoMap[location][i].Name < topoMap[location][j].Name }) leader := findLeader(topoMap[location]) - if location == localLocation && topoMap[location][leader].Name == hostname { + if location.nodeLocation != "" || (location.location == localLocation && topoMap[location][leader].Name == hostname) { t.leader = true } var allowedIPs []net.IPNet @@ -181,7 +200,8 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra endpoint: topoMap[location][leader].Endpoint, key: topoMap[location][leader].Key, persistentKeepalive: topoMap[location][leader].PersistentKeepalive, - location: location, + location: location.location, + nodeLocation: location.nodeLocation, cidrs: cidrs, hostnames: hostnames, leader: leader, @@ -225,7 +245,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra // Now that the topology is ordered, update the discoveredEndpoints map // add new ones by going through the ordered topology: segments, nodes - for _, node := range topoMap[segment.location] { + for _, node := range topoMap[topoKey{location: segment.location, nodeLocation: segment.nodeLocation}] { for key := range node.DiscoveredEndpoints { if _, ok := t.discoveredEndpoints[key]; !ok { t.discoveredEndpoints[key] = node.DiscoveredEndpoints[key] @@ -308,7 +328,7 @@ func (t *Topology) Conf() *wireguard.Conf { }, } for _, s := range t.segments { - if s.location == t.location { + if (s.location == t.location) || (t.nodeLocation != "" && t.nodeLocation == s.nodeLocation) { continue } peer := wireguard.Peer{ diff --git a/pkg/mesh/topology_test.go b/pkg/mesh/topology_test.go index 2f83fca..2773b4c 100644 --- a/pkg/mesh/topology_test.go +++ b/pkg/mesh/topology_test.go @@ -532,6 +532,274 @@ func TestNewTopology(t *testing.T) { logger: log.NewNopLogger(), }, }, + { + name: "cross from a", + granularity: CrossGranularity, + hostname: nodes["a"].Name, + result: &Topology{ + hostname: nodes["a"].Name, + leader: true, + location: nodeLocationPrefix + nodes["a"].Name, + nodeLocation: logicalLocationPrefix + nodes["a"].Location, + subnet: nodes["a"].Subnet, + privateIP: nodes["a"].InternalIP, + wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)}, + segments: []*segment{ + { + allowedIPs: []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["a"].Endpoint, + key: nodes["a"].Key, + persistentKeepalive: nodes["a"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["a"].Name, + nodeLocation: logicalLocationPrefix + nodes["a"].Location, + cidrs: []*net.IPNet{nodes["a"].Subnet}, + hostnames: []string{"a"}, + privateIPs: []net.IP{nodes["a"].InternalIP.IP}, + wireGuardIP: w1, + }, + { + allowedIPs: []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["b"].Endpoint, + key: nodes["b"].Key, + persistentKeepalive: nodes["b"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["b"].Name, + nodeLocation: logicalLocationPrefix + nodes["b"].Location, + cidrs: []*net.IPNet{nodes["b"].Subnet}, + hostnames: []string{"b"}, + privateIPs: []net.IP{nodes["b"].InternalIP.IP}, + wireGuardIP: w2, + allowedLocationIPs: nodes["b"].AllowedLocationIPs, + }, + { + allowedIPs: []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["c"].Endpoint, + key: nodes["c"].Key, + persistentKeepalive: nodes["c"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["c"].Name, + nodeLocation: logicalLocationPrefix + nodes["c"].Location, + cidrs: []*net.IPNet{nodes["c"].Subnet}, + hostnames: []string{"c"}, + privateIPs: []net.IP{nodes["c"].InternalIP.IP}, + wireGuardIP: w3, + }, + { + allowedIPs: []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["d"].Endpoint, + key: nodes["d"].Key, + persistentKeepalive: nodes["d"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["d"].Name, + nodeLocation: logicalLocationPrefix + nodes["d"].Location, + cidrs: []*net.IPNet{nodes["d"].Subnet}, + hostnames: []string{"d"}, + privateIPs: nil, + wireGuardIP: w4, + }, + }, + peers: []*Peer{peers["a"], peers["b"]}, + logger: log.NewNopLogger(), + }, + }, + { + name: "cross from b", + granularity: CrossGranularity, + hostname: nodes["b"].Name, + result: &Topology{ + hostname: nodes["b"].Name, + leader: true, + location: nodeLocationPrefix + nodes["b"].Name, + nodeLocation: logicalLocationPrefix + nodes["b"].Location, + subnet: nodes["b"].Subnet, + privateIP: nodes["b"].InternalIP, + wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)}, + segments: []*segment{ + { + allowedIPs: []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["a"].Endpoint, + key: nodes["a"].Key, + persistentKeepalive: nodes["a"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["a"].Name, + nodeLocation: logicalLocationPrefix + nodes["a"].Location, + cidrs: []*net.IPNet{nodes["a"].Subnet}, + hostnames: []string{"a"}, + privateIPs: []net.IP{nodes["a"].InternalIP.IP}, + wireGuardIP: w1, + }, + { + allowedIPs: []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["b"].Endpoint, + key: nodes["b"].Key, + persistentKeepalive: nodes["b"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["b"].Name, + nodeLocation: logicalLocationPrefix + nodes["b"].Location, + cidrs: []*net.IPNet{nodes["b"].Subnet}, + hostnames: []string{"b"}, + privateIPs: []net.IP{nodes["b"].InternalIP.IP}, + wireGuardIP: w2, + allowedLocationIPs: nodes["b"].AllowedLocationIPs, + }, + { + allowedIPs: []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["c"].Endpoint, + key: nodes["c"].Key, + persistentKeepalive: nodes["c"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["c"].Name, + nodeLocation: logicalLocationPrefix + nodes["c"].Location, + cidrs: []*net.IPNet{nodes["c"].Subnet}, + hostnames: []string{"c"}, + privateIPs: []net.IP{nodes["c"].InternalIP.IP}, + wireGuardIP: w3, + }, + { + allowedIPs: []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["d"].Endpoint, + key: nodes["d"].Key, + persistentKeepalive: nodes["d"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["d"].Name, + nodeLocation: logicalLocationPrefix + nodes["d"].Location, + cidrs: []*net.IPNet{nodes["d"].Subnet}, + hostnames: []string{"d"}, + privateIPs: nil, + wireGuardIP: w4, + }, + }, + peers: []*Peer{peers["a"], peers["b"]}, + logger: log.NewNopLogger(), + }, + }, + { + name: "cross from c", + granularity: CrossGranularity, + hostname: nodes["c"].Name, + result: &Topology{ + hostname: nodes["c"].Name, + leader: true, + location: nodeLocationPrefix + nodes["c"].Name, + nodeLocation: logicalLocationPrefix + nodes["c"].Location, + subnet: nodes["c"].Subnet, + privateIP: nodes["c"].InternalIP, + wireGuardCIDR: &net.IPNet{IP: w3, Mask: net.CIDRMask(16, 32)}, + segments: []*segment{ + { + allowedIPs: []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["a"].Endpoint, + key: nodes["a"].Key, + persistentKeepalive: nodes["a"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["a"].Name, + nodeLocation: logicalLocationPrefix + nodes["a"].Location, + cidrs: []*net.IPNet{nodes["a"].Subnet}, + hostnames: []string{"a"}, + privateIPs: []net.IP{nodes["a"].InternalIP.IP}, + wireGuardIP: w1, + }, + { + allowedIPs: []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["b"].Endpoint, + key: nodes["b"].Key, + persistentKeepalive: nodes["b"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["b"].Name, + nodeLocation: logicalLocationPrefix + nodes["b"].Location, + cidrs: []*net.IPNet{nodes["b"].Subnet}, + hostnames: []string{"b"}, + privateIPs: []net.IP{nodes["b"].InternalIP.IP}, + wireGuardIP: w2, + allowedLocationIPs: nodes["b"].AllowedLocationIPs, + }, + { + allowedIPs: []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["c"].Endpoint, + key: nodes["c"].Key, + persistentKeepalive: nodes["c"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["c"].Name, + nodeLocation: logicalLocationPrefix + nodes["c"].Location, + cidrs: []*net.IPNet{nodes["c"].Subnet}, + hostnames: []string{"c"}, + privateIPs: []net.IP{nodes["c"].InternalIP.IP}, + wireGuardIP: w3, + }, + { + allowedIPs: []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["d"].Endpoint, + key: nodes["d"].Key, + persistentKeepalive: nodes["d"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["d"].Name, + nodeLocation: logicalLocationPrefix + nodes["d"].Location, + cidrs: []*net.IPNet{nodes["d"].Subnet}, + hostnames: []string{"d"}, + privateIPs: nil, + wireGuardIP: w4, + }, + }, + peers: []*Peer{peers["a"], peers["b"]}, + logger: log.NewNopLogger(), + }, + }, + { + name: "cross from d", + granularity: CrossGranularity, + hostname: nodes["d"].Name, + result: &Topology{ + hostname: nodes["d"].Name, + leader: true, + location: nodeLocationPrefix + nodes["d"].Name, + nodeLocation: logicalLocationPrefix + nodes["d"].Location, + subnet: nodes["d"].Subnet, + privateIP: nil, + wireGuardCIDR: &net.IPNet{IP: w4, Mask: net.CIDRMask(16, 32)}, + segments: []*segment{ + { + allowedIPs: []net.IPNet{*nodes["a"].Subnet, *nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["a"].Endpoint, + key: nodes["a"].Key, + persistentKeepalive: nodes["a"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["a"].Name, + nodeLocation: logicalLocationPrefix + nodes["a"].Location, + cidrs: []*net.IPNet{nodes["a"].Subnet}, + hostnames: []string{"a"}, + privateIPs: []net.IP{nodes["a"].InternalIP.IP}, + wireGuardIP: w1, + }, + { + allowedIPs: []net.IPNet{*nodes["b"].Subnet, *nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["b"].Endpoint, + key: nodes["b"].Key, + persistentKeepalive: nodes["b"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["b"].Name, + nodeLocation: logicalLocationPrefix + nodes["b"].Location, + cidrs: []*net.IPNet{nodes["b"].Subnet}, + hostnames: []string{"b"}, + privateIPs: []net.IP{nodes["b"].InternalIP.IP}, + wireGuardIP: w2, + allowedLocationIPs: nodes["b"].AllowedLocationIPs, + }, + { + allowedIPs: []net.IPNet{*nodes["c"].Subnet, *nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["c"].Endpoint, + key: nodes["c"].Key, + persistentKeepalive: nodes["c"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["c"].Name, + nodeLocation: logicalLocationPrefix + nodes["c"].Location, + cidrs: []*net.IPNet{nodes["c"].Subnet}, + hostnames: []string{"c"}, + privateIPs: []net.IP{nodes["c"].InternalIP.IP}, + wireGuardIP: w3, + }, + { + allowedIPs: []net.IPNet{*nodes["d"].Subnet, {IP: w4, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["d"].Endpoint, + key: nodes["d"].Key, + persistentKeepalive: nodes["d"].PersistentKeepalive, + location: nodeLocationPrefix + nodes["d"].Name, + nodeLocation: logicalLocationPrefix + nodes["d"].Location, + cidrs: []*net.IPNet{nodes["d"].Subnet}, + hostnames: []string{"d"}, + privateIPs: nil, + wireGuardIP: w4, + }, + }, + peers: []*Peer{peers["a"], peers["b"]}, + logger: log.NewNopLogger(), + }, + }, } { tc.result.key = key tc.result.port = port