Merge pull request #292 from clive-jevons/bump-referenced-image-tag-to-0.4.1

bump referenced image version to 0.4.1 in preparation for release 0.4.1

.dockerignore

@@ -0,0 +1,3 @@
+**
+
+!/bin/linux

.github/workflows/ci.yml

@@ -3,14 +3,28 @@ name: CI
 on:
   push:
     branches: [ main ]
+    tags:
+      - "*"
   pull_request:
-    branches: [ main ]
   schedule:
   - cron:  '0 0 * * *'
   workflow_dispatch:
 
 jobs:
 
+  vendor:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17.1
+    - name: Vendor
+      run: |
+        make vendor
+        git diff --exit-code
+
   build:
     runs-on: ubuntu-latest
     steps:
@@ -18,7 +32,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Build
       run: make
 
@@ -29,7 +43,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Build kg and kgctl for all Linux Architectures
       run: make all-build
 
@@ -40,9 +54,11 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
-    - name: Build kgctl for Darwin
-      run: make OS=darwin
+        go-version: 1.17.1
+    - name: Build kgctl for Darwin amd64
+      run: make OS=darwin ARCH=amd64
+    - name: Build kgctl for Darwin arm64
+      run: make OS=darwin ARCH=arm64
 
   windows:
     runs-on: ubuntu-latest
@@ -51,7 +67,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Build kgctl for Windows
       run: make OS=windows
 
@@ -62,10 +78,22 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Run Unit Tests
       run: make unit
 
+  e2e:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17.1
+    - name: Run e2e Tests
+      run: make e2e
+
   lint:
     runs-on: ubuntu-latest
     steps:
@@ -73,7 +101,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Lint Code
       run: make lint
 
@@ -84,7 +112,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Enable Experimental Docker CLI
       run: |
         echo $'{\n  "experimental": true\n}' | sudo tee /etc/docker/daemon.json
@@ -100,6 +128,7 @@ jobs:
   push:
     if: github.event_name != 'pull_request'
     needs:
+    - vendor
     - build
     - linux
     - darwin
@@ -113,7 +142,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Enable Experimental Docker CLI
       run: |
         echo $'{\n  "experimental": true\n}' | sudo tee /etc/docker/daemon.json

.github/workflows/release.yaml

@@ -10,7 +10,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15.7
+        go-version: 1.17.1
     - name: Make Directory with kgctl Binaries to Be Released
       run: make release
     - name: Publish Release

.gitignore

@@ -4,3 +4,4 @@
 .push*
 bin/
 tmp/
+e2e/kind.yaml*

Dockerfile

@@ -1,15 +1,17 @@
 ARG FROM=alpine
-FROM alpine AS cni
-ARG GOARCH
+FROM $FROM AS cni
+ARG GOARCH=amd64
+ARG CNI_PLUGINS_VERSION=v0.9.1
 RUN apk add --no-cache curl && \
-    curl -Lo cni.tar.gz https://github.com/containernetworking/plugins/releases/download/v0.7.5/cni-plugins-$GOARCH-v0.7.5.tgz && \
+    curl -Lo cni.tar.gz https://github.com/containernetworking/plugins/releases/download/$CNI_PLUGINS_VERSION/cni-plugins-linux-$GOARCH-$CNI_PLUGINS_VERSION.tgz && \
     tar -xf cni.tar.gz
 
 FROM $FROM
 ARG GOARCH
+ARG ALPINE_VERSION=v3.12
 LABEL maintainer="squat <lserven@gmail.com>"
-RUN echo -e "https://alpine.global.ssl.fastly.net/alpine/v3.12/main\nhttps://alpine.global.ssl.fastly.net/alpine/v3.12/community" > /etc/apk/repositories && \
-    apk add --no-cache ipset iptables ip6tables wireguard-tools
+RUN echo -e "https://alpine.global.ssl.fastly.net/alpine/$ALPINE_VERSION/main\nhttps://alpine.global.ssl.fastly.net/alpine/$ALPINE_VERSION/community" > /etc/apk/repositories && \
+    apk add --no-cache ipset iptables ip6tables graphviz font-noto
 COPY --from=cni bridge host-local loopback portmap /opt/cni/bin/
 COPY bin/linux/$GOARCH/kg /opt/bin/
 ENTRYPOINT ["/opt/bin/kg"]

Makefile

@@ -1,9 +1,8 @@
 export GO111MODULE=on
-.PHONY: push container clean container-name container-latest push-latest fmt lint test unit vendor header generate client deepcopy informer lister openapi manifest manfest-latest manifest-annotate manifest manfest-latest manifest-annotate release
+.PHONY: push container clean container-name container-latest push-latest fmt lint test unit vendor header generate crd client deepcopy informer lister manifest manfest-latest manifest-annotate manifest manfest-latest manifest-annotate release gen-docs e2e
 
 OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
-ALL_OS := linux darwin windows
 ALL_ARCH := amd64 arm arm64
 DOCKER_ARCH := "amd64" "arm v7" "arm64 v8"
 ifeq ($(OS),linux)
@@ -11,12 +10,12 @@ ifeq ($(OS),linux)
 else
     BINS := bin/$(OS)/$(ARCH)/kgctl
 endif
-RELEASE_BINS := $(addprefix bin/release/kgctl-, $(addprefix linux-, $(ALL_ARCH)) darwin-amd64 windows-amd64)
-CLIENT_BINS := $(addsuffix /kgctl, $(addprefix bin/, $(addprefix linux/, $(ALL_ARCH)) darwin/amd64 windows/amd64))
+RELEASE_BINS := $(addprefix bin/release/kgctl-, $(addprefix linux-, $(ALL_ARCH)) darwin-amd64 darwin-arm64 windows-amd64)
 PROJECT := kilo
 PKG := github.com/squat/$(PROJECT)
 REGISTRY ?= index.docker.io
 IMAGE ?= squat/$(PROJECT)
+FULLY_QUALIFIED_IMAGE := $(REGISTRY)/$(IMAGE)
 
 TAG := $(shell git describe --abbrev=0 --tags HEAD 2>/dev/null)
 COMMIT := $(shell git rev-parse HEAD)
@@ -33,16 +32,21 @@ SRC := $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 GO_FILES ?= $$(find . -name '*.go' -not -path './vendor/*')
 GO_PKGS ?= $$(go list ./... | grep -v "$(PKG)/vendor")
 
+CONTROLLER_GEN_BINARY := bin/controller-gen
 CLIENT_GEN_BINARY := bin/client-gen
+DOCS_GEN_BINARY := bin/docs-gen
 DEEPCOPY_GEN_BINARY := bin/deepcopy-gen
 INFORMER_GEN_BINARY := bin/informer-gen
 LISTER_GEN_BINARY := bin/lister-gen
-OPENAPI_GEN_BINARY := bin/openapi-gen
 GOLINT_BINARY := bin/golint
 EMBEDMD_BINARY := bin/embedmd
+KIND_BINARY := $(shell pwd)/bin/kind
+KUBECTL_BINARY := $(shell pwd)/bin/kubectl
+BASH_UNIT := $(shell pwd)/bin/bash_unit
+BASH_UNIT_FLAGS :=
 
-BUILD_IMAGE ?= golang:1.15.7-alpine
-BASE_IMAGE ?= alpine:3.12
+BUILD_IMAGE ?= golang:1.17.1-alpine3.14
+BASE_IMAGE ?= alpine:3.14
 
 build: $(BINS)
 
@@ -71,7 +75,13 @@ all-container-latest: $(addprefix container-latest-, $(ALL_ARCH))
 
 all-push-latest: $(addprefix push-latest-, $(ALL_ARCH))
 
-generate: client deepcopy informer lister openapi
+generate: client deepcopy informer lister crd
+
+crd: manifests/crds.yaml
+manifests/crds.yaml: pkg/k8s/apis/kilo/v1alpha1/types.go $(CONTROLLER_GEN_BINARY)
+	$(CONTROLLER_GEN_BINARY) crd \
+	paths=./pkg/k8s/apis/kilo/... \
+	output:crd:stdout | tail -n +3 > $@
 
 client: pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go
 pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go: .header pkg/k8s/apis/kilo/v1alpha1/types.go $(CLIENT_GEN_BINARY)
@@ -129,16 +139,9 @@ pkg/k8s/listers/kilo/v1alpha1/peer.go: .header pkg/k8s/apis/kilo/v1alpha1/types.
 	rm -r github.com || true
 	go fmt ./pkg/k8s/listers/...
 
-openapi: pkg/k8s/apis/kilo/v1alpha1/openapi_generated.go
-pkg/k8s/apis/kilo/v1alpha1/openapi_generated.go: pkg/k8s/apis/kilo/v1alpha1/types.go $(OPENAPI_GEN_BINARY)
-	$(OPENAPI_GEN_BINARY) \
-	--input-dirs $(PKG)/$(@D),k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/core/v1 \
-	--output-base $(CURDIR) \
-	--output-package ./$(@D) \
-	--logtostderr \
-	--report-filename /dev/null \
-	--go-header-file=.header
-	go fmt $@
+gen-docs: generate docs/api.md
+docs/api.md: pkg/k8s/apis/kilo/v1alpha1/types.go $(DOCS_GEN_BINARY)
+	$(DOCS_GEN_BINARY) $< > $@
 
 $(BINS): $(SRC) go.mod
 	@mkdir -p bin/$(word 2,$(subst /, ,$@))/$(word 3,$(subst /, ,$@))
@@ -191,7 +194,22 @@ lint: header $(GOLINT_BINARY)
 unit:
 	go test -mod=vendor --race ./...
 
-test: lint unit
+test: lint unit e2e
+
+$(KIND_BINARY):
+	curl -Lo $@ https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-$(ARCH)
+	chmod +x $@
+
+$(KUBECTL_BINARY):
+	curl -Lo $@ https://dl.k8s.io/release/v1.21.0/bin/linux/$(ARCH)/kubectl
+	chmod +x $@
+
+$(BASH_UNIT):
+	curl -Lo $@ https://raw.githubusercontent.com/pgrange/bash_unit/v1.7.2/bash_unit
+	chmod +x $@
+
+e2e: container $(KIND_BINARY) $(KUBECTL_BINARY) $(BASH_UNIT) bin/$(OS)/$(ARCH)/kgctl
+	KILO_IMAGE=$(IMAGE):$(ARCH)-$(VERSION) KIND_BINARY=$(KIND_BINARY) KUBECTL_BINARY=$(KUBECTL_BINARY) KGCTL_BINARY=$(shell pwd)/bin/$(OS)/$(ARCH)/kgctl $(BASH_UNIT) $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/multi-cluster.sh ./e2e/handlers.sh ./e2e/kgctl.sh ./e2e/teardown.sh
 
 header: .header
 	@HEADER=$$(cat .header); \
@@ -224,10 +242,12 @@ website/docs/README.md: README.md
 	cat README.md >> $@
 	cp -r docs/graphs website/static/img/
 	sed -i 's/\.\/docs\///g' $@
-	find $(@D)  -type f -name '*.md' | xargs -I{} sed -i 's/\.\/\(.\+\.svg\)/\/img\/\1/g' {}
+	find $(@D)  -type f -name '*.md' | xargs -I{} sed -i 's/\.\/\(.\+\.\(svg\|png\)\)/\/img\/\1/g' {}
 	sed -i 's/graphs\//\/img\/graphs\//g' $@
+	# The next line is a workaround until mdx, docusaurus' markdown parser, can parse links with preceding brackets.
+	sed -i  's/\[\]\(\[.*\](.*)\)/\&#91;\&#93;\1/g' website/docs/api.md
 
-website/build/index.html: website/docs/README.md
+website/build/index.html: website/docs/README.md docs/api.md
 	yarn --cwd website install
 	yarn --cwd website build
 
@@ -243,7 +263,7 @@ container: .container-$(ARCH)-$(VERSION) container-name
 	@docker images -q $(IMAGE):$(ARCH)-$(VERSION) > $@
 
 container-latest: .container-$(ARCH)-$(VERSION)
-	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(IMAGE):$(ARCH)-latest
+	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(FULLY_QUALIFIED_IMAGE):$(ARCH)-latest
 	@echo "container: $(IMAGE):$(ARCH)-latest"
 
 container-name:
@@ -251,14 +271,15 @@ container-name:
 
 manifest: .manifest-$(VERSION) manifest-name
 .manifest-$(VERSION): Dockerfile $(addprefix push-, $(ALL_ARCH))
-	@docker manifest create --amend $(IMAGE):$(VERSION) $(addsuffix -$(VERSION), $(addprefix squat/$(PROJECT):, $(ALL_ARCH)))
+	@docker manifest create --amend $(FULLY_QUALIFIED_IMAGE):$(VERSION) $(addsuffix -$(VERSION), $(addprefix $(FULLY_QUALIFIED_IMAGE):, $(ALL_ARCH)))
 	@$(MAKE) --no-print-directory manifest-annotate-$(VERSION)
-	@docker manifest push $(IMAGE):$(VERSION) > $@
+	@docker manifest push $(FULLY_QUALIFIED_IMAGE):$(VERSION) > $@
 
 manifest-latest: Dockerfile $(addprefix push-latest-, $(ALL_ARCH))
-	@docker manifest create --amend $(IMAGE):latest $(addsuffix -latest, $(addprefix squat/$(PROJECT):, $(ALL_ARCH)))
+	@docker manifest rm $(FULLY_QUALIFIED_IMAGE):latest || echo no old manifest
+	@docker manifest create --amend $(FULLY_QUALIFIED_IMAGE):latest $(addsuffix -latest, $(addprefix $(FULLY_QUALIFIED_IMAGE):, $(ALL_ARCH)))
 	@$(MAKE) --no-print-directory manifest-annotate-latest
-	@docker manifest push $(IMAGE):latest
+	@docker manifest push $(FULLY_QUALIFIED_IMAGE):latest
 	@echo "manifest: $(IMAGE):latest"
 
 manifest-annotate: manifest-annotate-$(VERSION)
@@ -269,7 +290,7 @@ manifest-annotate-%:
 	    annotate=; \
 	    j=0; for da in $(DOCKER_ARCH); do \
 		if [ "$$j" -eq "$$i" ] && [ -n "$$da" ]; then \
-		    annotate="docker manifest annotate $(IMAGE):$* $(IMAGE):$$a-$* --os linux --arch"; \
+		    annotate="docker manifest annotate $(FULLY_QUALIFIED_IMAGE):$* $(FULLY_QUALIFIED_IMAGE):$$a-$* --os linux --arch"; \
 		    k=0; for ea in $$da; do \
 			[ "$$k" = 0 ] && annotate="$$annotate $$ea"; \
 			[ "$$k" != 0 ] && annotate="$$annotate --variant $$ea"; \
@@ -283,15 +304,18 @@ manifest-annotate-%:
 	done
 
 manifest-name:
-	@echo "manifest: $(IMAGE_ROOT):$(VERSION)"
+	@echo "manifest: $(IMAGE):$(VERSION)"
 
 push: .push-$(ARCH)-$(VERSION) push-name
 .push-$(ARCH)-$(VERSION): .container-$(ARCH)-$(VERSION)
-	@docker push $(REGISTRY)/$(IMAGE):$(ARCH)-$(VERSION)
+ifneq ($(REGISTRY),index.docker.io)
+	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(FULLY_QUALIFIED_IMAGE):$(ARCH)-$(VERSION)
+endif
+	@docker push $(FULLY_QUALIFIED_IMAGE):$(ARCH)-$(VERSION)
 	@docker images -q $(IMAGE):$(ARCH)-$(VERSION) > $@
 
 push-latest: container-latest
-	@docker push $(REGISTRY)/$(IMAGE):$(ARCH)-latest
+	@docker push $(FULLY_QUALIFIED_IMAGE):$(ARCH)-latest
 	@echo "pushed: $(IMAGE):$(ARCH)-latest"
 
 push-name:
@@ -316,6 +340,9 @@ vendor:
 	go mod tidy
 	go mod vendor
 
+$(CONTROLLER_GEN_BINARY):
+	go build -mod=vendor -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen
+
 $(CLIENT_GEN_BINARY):
 	go build -mod=vendor -o $@ k8s.io/code-generator/cmd/client-gen
 
@@ -328,8 +355,8 @@ $(INFORMER_GEN_BINARY):
 $(LISTER_GEN_BINARY):
 	go build -mod=vendor -o $@ k8s.io/code-generator/cmd/lister-gen
 
-$(OPENAPI_GEN_BINARY):
-	go build -mod=vendor -o $@ k8s.io/kube-openapi/cmd/openapi-gen
+$(DOCS_GEN_BINARY): cmd/docs-gen/main.go
+	go build -mod=vendor -o $@ ./cmd/docs-gen
 
 $(GOLINT_BINARY):
 	go build -mod=vendor -o $@ golang.org/x/lint/golint

README.md

@@ -6,17 +6,20 @@ Kilo is a multi-cloud network overlay built on WireGuard and designed for Kubern
 
 [![Build Status](https://github.com/squat/kilo/workflows/CI/badge.svg)](https://github.com/squat/kilo/actions?query=workflow%3ACI)
 [![Go Report Card](https://goreportcard.com/badge/github.com/squat/kilo)](https://goreportcard.com/report/github.com/squat/kilo)
+[![Docker Pulls](https://img.shields.io/docker/pulls/squat/kilo)](https://hub.docker.com/r/squat/kilo)
+[![Slack](https://img.shields.io/badge/join%20slack-%23kilo-brightgreen.svg)](https://slack.k8s.io/)
 
 ## Overview
 
 Kilo connects nodes in a cluster by providing an encrypted layer 3 network that can span across data centers and public clouds.
+The Pod network created by Kilo is always fully connected, even when the nodes are in different networks or behind NAT.
 By allowing pools of nodes in different locations to communicate securely, Kilo enables the operation of multi-cloud clusters.
 Kilo's design allows clients to VPN to a cluster in order to securely access services running on the cluster.
 In addition to creating multi-cloud clusters, Kilo enables the creation of multi-cluster services, i.e. services that span across different Kubernetes clusters.
 
 An introductory video about Kilo from KubeCon EU 2019 can be found on [youtube](https://www.youtube.com/watch?v=iPz_DAOOCKA).
 
-## How it works
+## How It Works
 
 Kilo uses [WireGuard](https://www.wireguard.com/), a performant and secure VPN, to create a mesh between the different nodes in a cluster.
 The Kilo agent, `kg`, runs on every node in the cluster, setting up the public and private keys for the VPN as well as the necessary rules to route packets between locations.
@@ -69,24 +72,28 @@ Kilo can be installed by deploying a DaemonSet to the cluster.
 To run Kilo on kubeadm:
 
 ```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
 kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-kubeadm.yaml
 ```
 
 To run Kilo on bootkube:
 
 ```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
 kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-bootkube.yaml
 ```
 
 To run Kilo on Typhoon:
 
 ```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
 kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-typhoon.yaml
 ```
 
 To run Kilo on k3s:
 
 ```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
 kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s.yaml
 ```
 
@@ -99,6 +106,7 @@ Kilo currently supports running on top of Flannel.
 For example, to run Kilo on a Typhoon cluster running Flannel:
 
 ```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
 kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-typhoon-flannel.yaml
 ```
 
@@ -107,7 +115,7 @@ kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kil
 ## VPN
 
 Kilo also enables peers outside of a Kubernetes cluster to connect to the VPN, allowing cluster applications to securely access external services and permitting developers and support to securely debug cluster resources.
-In order to declare a peer, start by defining a Kilo peer resource:
+In order to declare a peer, start by defining a Kilo Peer resource:
 
 ```shell
 cat <<'EOF' | kubectl apply -f -
@@ -148,7 +156,7 @@ for n in $(kubectl --kubeconfig $KUBECONFIG2 get no -o name | cut -d'/' -f2); do
     kgctl --kubeconfig $KUBECONFIG2 showconf node $n --as-peer -o yaml --allowed-ips $SERVICECIDR2 | kubectl --kubeconfig $KUBECONFIG1 apply -f -
 done
 # Create a Service in cluster2 to mirror the Service in cluster1.
-cat <<'EOF' | kubectl --kubeconfig $KUBECONFIG2 apply -f -
+cat <<EOF | kubectl --kubeconfig $KUBECONFIG2 apply -f -
 apiVersion: v1
 kind: Service
 metadata:

cmd/docs-gen/main.go

@@ -0,0 +1,292 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file was adapted from https://github.com/prometheus-operator/prometheus-operator/blob/master/cmd/po-docgen/api.go.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"os"
+	"reflect"
+	"strings"
+)
+
+const (
+	firstParagraph = `# API
+This document is a reference of the API types introduced by Kilo.
+
+> **Note**: this document is generated from code comments. When contributing a change to this document, please do so by changing the code comments.`
+)
+
+var (
+	links = map[string]string{
+		"metav1.ObjectMeta": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#objectmeta-v1-meta",
+		"metav1.ListMeta":   "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#listmeta-v1-meta",
+	}
+
+	selfLinks = map[string]string{}
+	typesDoc  = map[string]KubeTypes{}
+)
+
+func toSectionLink(name string) string {
+	name = strings.ToLower(name)
+	name = strings.Replace(name, " ", "-", -1)
+	return name
+}
+
+func printTOC(types []KubeTypes) {
+	fmt.Printf("\n## Table of Contents\n")
+	for _, t := range types {
+		strukt := t[0]
+		if len(t) > 1 {
+			fmt.Printf("* [%s](#%s)\n", strukt.Name, toSectionLink(strukt.Name))
+		}
+	}
+}
+
+func printAPIDocs(paths []string) {
+	fmt.Println(firstParagraph)
+
+	types := parseDocumentationFrom(paths)
+	for _, t := range types {
+		strukt := t[0]
+		selfLinks[strukt.Name] = "#" + strings.ToLower(strukt.Name)
+		typesDoc[toLink(strukt.Name)] = t[1:]
+	}
+
+	// we need to parse once more to now add the self links and the inlined fields
+	types = parseDocumentationFrom(paths)
+
+	printTOC(types)
+
+	for _, t := range types {
+		strukt := t[0]
+		if len(t) > 1 {
+			fmt.Printf("\n## %s\n\n%s\n\n", strukt.Name, strukt.Doc)
+
+			fmt.Println("| Field | Description | Scheme | Required |")
+			fmt.Println("| ----- | ----------- | ------ | -------- |")
+			fields := t[1:]
+			for _, f := range fields {
+				fmt.Println("|", f.Name, "|", f.Doc, "|", f.Type, "|", f.Mandatory, "|")
+			}
+			fmt.Println("")
+			fmt.Println("[Back to TOC](#table-of-contents)")
+		}
+	}
+}
+
+// Pair of strings. We need the name of fields and the doc.
+type Pair struct {
+	Name, Doc, Type string
+	Mandatory       bool
+}
+
+// KubeTypes is an array to represent all available types in a parsed file. [0] is for the type itself
+type KubeTypes []Pair
+
+// parseDocumentationFrom gets all types' documentation and returns them as an
+// array. Each type is again represented as an array (we have to use arrays as we
+// need to be sure of the order of the fields). This function returns fields and
+// struct definitions that have no documentation as {name, ""}.
+func parseDocumentationFrom(srcs []string) []KubeTypes {
+	var docForTypes []KubeTypes
+
+	for _, src := range srcs {
+		pkg := astFrom(src)
+
+		for _, kubType := range pkg.Types {
+			if structType, ok := kubType.Decl.Specs[0].(*ast.TypeSpec).Type.(*ast.StructType); ok {
+				var ks KubeTypes
+				ks = append(ks, Pair{kubType.Name, fmtRawDoc(kubType.Doc), "", false})
+
+				for _, field := range structType.Fields.List {
+					// Skip fields that are not tagged.
+					if field.Tag == nil {
+						os.Stderr.WriteString(fmt.Sprintf("Tag is nil, skipping field: %v of type %v\n", field, field.Type))
+						continue
+					}
+					// Treat inlined fields separately as we don't want the original types to appear in the doc.
+					if isInlined(field) {
+						// Skip external types, as we don't want their content to be part of the API documentation.
+						if isInternalType(field.Type) {
+							ks = append(ks, typesDoc[fieldType(field.Type)]...)
+						}
+						continue
+					}
+
+					typeString := fieldType(field.Type)
+					fieldMandatory := fieldRequired(field)
+					if n := fieldName(field); n != "-" {
+						fieldDoc := fmtRawDoc(field.Doc.Text())
+						ks = append(ks, Pair{n, fieldDoc, typeString, fieldMandatory})
+					}
+				}
+				docForTypes = append(docForTypes, ks)
+			}
+		}
+	}
+
+	return docForTypes
+}
+
+func astFrom(filePath string) *doc.Package {
+	fset := token.NewFileSet()
+	m := make(map[string]*ast.File)
+
+	f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments)
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+
+	m[filePath] = f
+	apkg, _ := ast.NewPackage(fset, m, nil, nil)
+
+	return doc.New(apkg, "", 0)
+}
+
+func fmtRawDoc(rawDoc string) string {
+	var buffer bytes.Buffer
+	delPrevChar := func() {
+		if buffer.Len() > 0 {
+			buffer.Truncate(buffer.Len() - 1) // Delete the last " " or "\n"
+		}
+	}
+
+	// Ignore all lines after ---
+	rawDoc = strings.Split(rawDoc, "---")[0]
+
+	for _, line := range strings.Split(rawDoc, "\n") {
+		line = strings.TrimRight(line, " ")
+		leading := strings.TrimLeft(line, " ")
+		switch {
+		case len(line) == 0: // Keep paragraphs
+			delPrevChar()
+			buffer.WriteString("\n\n")
+		case strings.HasPrefix(leading, "TODO"): // Ignore one line TODOs
+		case strings.HasPrefix(leading, "+"): // Ignore instructions to go2idl
+		default:
+			if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
+				delPrevChar()
+				line = "\n" + line + "\n" // Replace it with newline. This is useful when we have a line with: "Example:\n\tJSON-someting..."
+			} else {
+				line += " "
+			}
+			buffer.WriteString(line)
+		}
+	}
+
+	postDoc := strings.TrimRight(buffer.String(), "\n")
+	postDoc = strings.Replace(postDoc, "\\\"", "\"", -1) // replace user's \" to "
+	postDoc = strings.Replace(postDoc, "\"", "\\\"", -1) // Escape "
+	postDoc = strings.Replace(postDoc, "\n", "\\n", -1)
+	postDoc = strings.Replace(postDoc, "\t", "\\t", -1)
+	postDoc = strings.Replace(postDoc, "|", "\\|", -1)
+
+	return postDoc
+}
+
+func toLink(typeName string) string {
+	selfLink, hasSelfLink := selfLinks[typeName]
+	if hasSelfLink {
+		return wrapInLink(typeName, selfLink)
+	}
+
+	link, hasLink := links[typeName]
+	if hasLink {
+		return wrapInLink(typeName, link)
+	}
+
+	return typeName
+}
+
+func wrapInLink(text, link string) string {
+	return fmt.Sprintf("[%s](%s)", text, link)
+}
+
+func isInlined(field *ast.Field) bool {
+	jsonTag := reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+	return strings.Contains(jsonTag, "inline")
+}
+
+func isInternalType(typ ast.Expr) bool {
+	switch typ := typ.(type) {
+	case *ast.SelectorExpr:
+		pkg := typ.X.(*ast.Ident)
+		return strings.HasPrefix(pkg.Name, "monitoring")
+	case *ast.StarExpr:
+		return isInternalType(typ.X)
+	case *ast.ArrayType:
+		return isInternalType(typ.Elt)
+	case *ast.MapType:
+		return isInternalType(typ.Key) && isInternalType(typ.Value)
+	default:
+		return true
+	}
+}
+
+// fieldName returns the name of the field as it should appear in JSON format
+// "-" indicates that this field is not part of the JSON representation
+func fieldName(field *ast.Field) string {
+	jsonTag := reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+	jsonTag = strings.Split(jsonTag, ",")[0]                                              // This can return "-"
+	if jsonTag == "" {
+		if field.Names != nil {
+			return field.Names[0].Name
+		}
+		return field.Type.(*ast.Ident).Name
+	}
+	return jsonTag
+}
+
+// fieldRequired returns whether a field is a required field.
+func fieldRequired(field *ast.Field) bool {
+	jsonTag := ""
+	if field.Tag != nil {
+		jsonTag = reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+		return !strings.Contains(jsonTag, "omitempty")
+	}
+
+	return false
+}
+
+func fieldType(typ ast.Expr) string {
+	switch typ := typ.(type) {
+	case *ast.Ident:
+		return toLink(typ.Name)
+	case *ast.StarExpr:
+		return "*" + toLink(fieldType(typ.X))
+	case *ast.SelectorExpr:
+		pkg := typ.X.(*ast.Ident)
+		t := typ.Sel
+		return toLink(pkg.Name + "." + t.Name)
+	case *ast.ArrayType:
+		return "[]" + toLink(fieldType(typ.Elt))
+	case *ast.MapType:
+		return "map[" + toLink(fieldType(typ.Key)) + "]" + toLink(fieldType(typ.Value))
+	default:
+		return ""
+	}
+}
+
+func main() {
+	printAPIDocs(os.Args[1:])
+}

cmd/kg/handlers.go

@@ -0,0 +1,147 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/squat/kilo/pkg/mesh"
+)
+
+type graphHandler struct {
+	mesh        *mesh.Mesh
+	granularity mesh.Granularity
+	hostname    *string
+	subnet      *net.IPNet
+}
+
+func (h *graphHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ns, err := h.mesh.Nodes().List()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to list nodes: %v", err), http.StatusInternalServerError)
+		return
+	}
+	ps, err := h.mesh.Peers().List()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to list peers: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	nodes := make(map[string]*mesh.Node)
+	for _, n := range ns {
+		if n.Ready() {
+			nodes[n.Name] = n
+		}
+	}
+	if len(nodes) == 0 {
+		http.Error(w, "did not find any valid Kilo nodes in the cluster", http.StatusInternalServerError)
+		return
+	}
+	peers := make(map[string]*mesh.Peer)
+	for _, p := range ps {
+		if p.Ready() {
+			peers[p.Name] = p
+		}
+	}
+	topo, err := mesh.NewTopology(nodes, peers, h.granularity, *h.hostname, 0, wgtypes.Key{}, h.subnet, nodes[*h.hostname].PersistentKeepalive, nil)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to create topology: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	dot, err := topo.Dot()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to generate graph: %v", err), http.StatusInternalServerError)
+	}
+
+	buf := bytes.NewBufferString(dot)
+
+	format := r.URL.Query().Get("format")
+	switch format {
+	case "":
+		format = "svg"
+	case "dot", "gv":
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		w.Write(buf.Bytes())
+		return
+
+	case "svg", "png", "bmp", "fig", "gif", "json", "ps":
+		// Accepted format
+
+	default:
+		http.Error(w, "unsupported format", http.StatusInternalServerError)
+		return
+	}
+
+	layout := r.URL.Query().Get("layout")
+	switch layout {
+	case "":
+		layout = "circo"
+
+	case "circo", "dot", "neato", "twopi", "fdp":
+		// Accepted layout
+
+	default:
+		http.Error(w, "unsupported layout", http.StatusInternalServerError)
+		return
+	}
+
+	command := exec.Command("dot", "-K"+layout, "-T"+format)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if _, err = io.Copy(stdin, buf); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err = stdin.Close(); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	output, err := command.Output()
+	if err != nil {
+		http.Error(w, "unable to render graph", http.StatusInternalServerError)
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + format)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	w.Header().Add("content-type", mimeType)
+	w.Write(output)
+}
+
+func healthHandler(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}

cmd/kg/main.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -29,8 +29,9 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	flag "github.com/spf13/pflag"
+	"github.com/spf13/cobra"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -40,6 +41,7 @@ import (
 	kiloclient "github.com/squat/kilo/pkg/k8s/clientset/versioned"
 	"github.com/squat/kilo/pkg/mesh"
 	"github.com/squat/kilo/pkg/version"
+	"github.com/squat/kilo/pkg/wireguard"
 )
 
 const (
@@ -77,51 +79,79 @@ var (
 	}, ", ")
 )
 
-// Main is the principal function for the binary, wrapped only by `main` for convenience.
-func Main() error {
-	backend := flag.String("backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
-	cleanUpIface := flag.Bool("clean-up-interface", false, "Should Kilo delete its interface when it shuts down?")
-	createIface := flag.Bool("create-interface", true, "Should kilo create an interface on startup?")
-	cni := flag.Bool("cni", true, "Should Kilo manage the node's CNI configuration?")
-	cniPath := flag.String("cni-path", mesh.DefaultCNIPath, "Path to CNI config.")
-	compatibility := flag.String("compatibility", "", fmt.Sprintf("Should Kilo run in compatibility mode? Possible values: %s", availableCompatibilities))
-	encapsulate := flag.String("encapsulate", string(encapsulation.Always), fmt.Sprintf("When should Kilo encapsulate packets within a location? Possible values: %s", availableEncapsulations))
-	granularity := flag.String("mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
-	kubeconfig := flag.String("kubeconfig", "", "Path to kubeconfig.")
-	hostname := flag.String("hostname", "", "Hostname of the node on which this process is running.")
-	iface := flag.String("interface", mesh.DefaultKiloInterface, "Name of the Kilo interface to use; if it does not exist, it will be created.")
-	listen := flag.String("listen", ":1107", "The address at which to listen for health and metrics.")
-	local := flag.Bool("local", true, "Should Kilo manage routes within a location?")
-	logLevel := flag.String("log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels))
-	master := flag.String("master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
-	topologyLabel := flag.String("topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
-	var port uint
-	flag.UintVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
-	subnet := flag.String("subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.")
-	resyncPeriod := flag.Duration("resync-period", 30*time.Second, "How often should the Kilo controllers reconcile?")
-	printVersion := flag.Bool("version", false, "Print version and exit")
-	flag.Parse()
+var cmd = &cobra.Command{
+	Use:   "kg",
+	Short: "kg is the Kilo agent",
+	Long: `kg is the Kilo agent.
+		It runs on every node of a cluster,
+		setting up the public and private keys for the VPN
+		as well as the necessary rules to route packets between locations.`,
+	PreRunE:       preRun,
+	RunE:          runRoot,
+	SilenceUsage:  true,
+	SilenceErrors: true,
+}
 
-	if *printVersion {
-		fmt.Println(version.Version)
-		return nil
-	}
+var (
+	backend               string
+	cleanUpIface          bool
+	createIface           bool
+	cni                   bool
+	cniPath               string
+	compatibility         string
+	encapsulate           string
+	granularity           string
+	hostname              string
+	kubeconfig            string
+	iface                 string
+	listen                string
+	local                 bool
+	master                string
+	mtu                   uint
+	topologyLabel         string
+	port                  int
+	subnet                string
+	resyncPeriod          time.Duration
+	iptablesForwardRule   bool
+	prioritisePrivateAddr bool
 
-	_, s, err := net.ParseCIDR(*subnet)
-	if err != nil {
-		return fmt.Errorf("failed to parse %q as CIDR: %v", *subnet, err)
-	}
+	printVersion bool
+	logLevel     string
 
-	if *hostname == "" {
-		var err error
-		*hostname, err = os.Hostname()
-		if *hostname == "" || err != nil {
-			return errors.New("failed to determine hostname")
-		}
-	}
+	logger   log.Logger
+	registry *prometheus.Registry
+)
 
-	logger := log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
-	switch *logLevel {
+func init() {
+	cmd.Flags().StringVar(&backend, "backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
+	cmd.Flags().BoolVar(&cleanUpIface, "clean-up-interface", false, "Should Kilo delete its interface when it shuts down?")
+	cmd.Flags().BoolVar(&createIface, "create-interface", true, "Should kilo create an interface on startup?")
+	cmd.Flags().BoolVar(&cni, "cni", true, "Should Kilo manage the node's CNI configuration?")
+	cmd.Flags().StringVar(&cniPath, "cni-path", mesh.DefaultCNIPath, "Path to CNI config.")
+	cmd.Flags().StringVar(&compatibility, "compatibility", "", fmt.Sprintf("Should Kilo run in compatibility mode? Possible values: %s", availableCompatibilities))
+	cmd.Flags().StringVar(&encapsulate, "encapsulate", string(encapsulation.Always), fmt.Sprintf("When should Kilo encapsulate packets within a location? Possible values: %s", availableEncapsulations))
+	cmd.Flags().StringVar(&granularity, "mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
+	cmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", "Path to kubeconfig.")
+	cmd.Flags().StringVar(&hostname, "hostname", "", "Hostname of the node on which this process is running.")
+	cmd.Flags().StringVar(&iface, "interface", mesh.DefaultKiloInterface, "Name of the Kilo interface to use; if it does not exist, it will be created.")
+	cmd.Flags().StringVar(&listen, "listen", ":1107", "The address at which to listen for health and metrics.")
+	cmd.Flags().BoolVar(&local, "local", true, "Should Kilo manage routes within a location?")
+	cmd.Flags().StringVar(&master, "master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
+	cmd.Flags().UintVar(&mtu, "mtu", wireguard.DefaultMTU, "The MTU of the WireGuard interface created by Kilo.")
+	cmd.Flags().StringVar(&topologyLabel, "topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
+	cmd.Flags().IntVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
+	cmd.Flags().StringVar(&subnet, "subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.")
+	cmd.Flags().DurationVar(&resyncPeriod, "resync-period", 30*time.Second, "How often should the Kilo controllers reconcile?")
+	cmd.Flags().BoolVar(&iptablesForwardRule, "iptables-forward-rules", false, "Add default accept rules to the FORWARD chain in iptables. Warning: this may break firewalls with a deny all policy and is potentially insecure!")
+	cmd.Flags().BoolVar(&prioritisePrivateAddr, "prioritise-private-addresses", false, "Prefer to assign a private IP address to the node's endpoint.")
+
+	cmd.PersistentFlags().BoolVar(&printVersion, "version", false, "Print version and exit")
+	cmd.PersistentFlags().StringVar(&logLevel, "log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels))
+}
+
+func preRun(_ *cobra.Command, _ []string) error {
+	logger = log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
+	switch logLevel {
 	case logLevelAll:
 		logger = level.NewFilter(logger, level.AllowAll())
 	case logLevelDebug:
@@ -135,74 +165,100 @@ func Main() error {
 	case logLevelNone:
 		logger = level.NewFilter(logger, level.AllowNone())
 	default:
-		return fmt.Errorf("log level %v unknown; possible values are: %s", *logLevel, availableLogLevels)
+		return fmt.Errorf("log level %v unknown; possible values are: %s", logLevel, availableLogLevels)
 	}
 	logger = log.With(logger, "ts", log.DefaultTimestampUTC)
 	logger = log.With(logger, "caller", log.DefaultCaller)
 
-	e := encapsulation.Strategy(*encapsulate)
+	registry = prometheus.NewRegistry()
+	registry.MustRegister(
+		collectors.NewGoCollector(),
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+	)
+
+	return nil
+}
+
+// runRoot is the principal function for the binary.
+func runRoot(_ *cobra.Command, _ []string) error {
+	if printVersion {
+		fmt.Println(version.Version)
+		return nil
+	}
+
+	_, s, err := net.ParseCIDR(subnet)
+	if err != nil {
+		return fmt.Errorf("failed to parse %q as CIDR: %v", subnet, err)
+	}
+
+	if hostname == "" {
+		var err error
+		hostname, err = os.Hostname()
+		if hostname == "" || err != nil {
+			return errors.New("failed to determine hostname")
+		}
+	}
+
+	e := encapsulation.Strategy(encapsulate)
 	switch e {
 	case encapsulation.Never:
 	case encapsulation.CrossSubnet:
 	case encapsulation.Always:
 	default:
-		return fmt.Errorf("encapsulation %v unknown; possible values are: %s", *encapsulate, availableEncapsulations)
+		return fmt.Errorf("encapsulation %v unknown; possible values are: %s", encapsulate, availableEncapsulations)
 	}
 
 	var enc encapsulation.Encapsulator
-	switch *compatibility {
+	switch compatibility {
 	case "flannel":
 		enc = encapsulation.NewFlannel(e)
 	default:
 		enc = encapsulation.NewIPIP(e)
 	}
 
-	gr := mesh.Granularity(*granularity)
+	gr := mesh.Granularity(granularity)
 	switch gr {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
 	default:
-		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", *granularity, availableGranularities)
+		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", granularity, availableGranularities)
 	}
 
 	var b mesh.Backend
-	switch *backend {
+	switch backend {
 	case k8s.Backend:
-		config, err := clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
+		config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig)
 		if err != nil {
 			return fmt.Errorf("failed to create Kubernetes config: %v", err)
 		}
 		c := kubernetes.NewForConfigOrDie(config)
 		kc := kiloclient.NewForConfigOrDie(config)
 		ec := apiextensions.NewForConfigOrDie(config)
-		b = k8s.New(c, kc, ec, *topologyLabel)
+		b = k8s.New(c, kc, ec, topologyLabel, log.With(logger, "component", "k8s backend"))
 	default:
-		return fmt.Errorf("backend %v unknown; possible values are: %s", *backend, availableBackends)
+		return fmt.Errorf("backend %v unknown; possible values are: %s", backend, availableBackends)
 	}
 
-	m, err := mesh.New(b, enc, gr, *hostname, uint32(port), s, *local, *cni, *cniPath, *iface, *cleanUpIface, *createIface, *resyncPeriod, log.With(logger, "component", "kilo"))
+	if port < 1 || port > 1<<16-1 {
+		return fmt.Errorf("invalid port: port mus be in range [%d:%d], but got %d", 1, 1<<16-1, port)
+	}
+	m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, log.With(logger, "component", "kilo"))
 	if err != nil {
 		return fmt.Errorf("failed to create Kilo mesh: %v", err)
 	}
 
-	r := prometheus.NewRegistry()
-	r.MustRegister(
-		prometheus.NewGoCollector(),
-		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
-	)
-	m.RegisterMetrics(r)
+	m.RegisterMetrics(registry)
 
 	var g run.Group
 	{
 		// Run the HTTP server.
 		mux := http.NewServeMux()
-		mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
-			w.WriteHeader(http.StatusOK)
-		})
-		mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{}))
-		l, err := net.Listen("tcp", *listen)
+		mux.HandleFunc("/health", healthHandler)
+		mux.Handle("/graph", &graphHandler{m, gr, &hostname, s})
+		mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+		l, err := net.Listen("tcp", listen)
 		if err != nil {
-			return fmt.Errorf("failed to listen on %s: %v", *listen, err)
+			return fmt.Errorf("failed to listen on %s: %v", listen, err)
 		}
 
 		g.Add(func() error {
@@ -251,8 +307,15 @@ func Main() error {
 	return g.Run()
 }
 
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print the version and exit.",
+	Run:   func(_ *cobra.Command, _ []string) { fmt.Println(version.Version) },
+}
+
 func main() {
-	if err := Main(); err != nil {
+	cmd.AddCommand(webhookCmd, versionCmd)
+	if err := cmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}

cmd/kg/webhook.go

@@ -0,0 +1,273 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/go-kit/kit/log/level"
+	"github.com/oklog/run"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/cobra"
+	v1 "k8s.io/api/admission/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+
+	kilo "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
+	"github.com/squat/kilo/pkg/version"
+)
+
+var webhookCmd = &cobra.Command{
+	Use: "webhook",
+	PreRunE: func(c *cobra.Command, a []string) error {
+		if c.HasParent() {
+			return c.Parent().PreRunE(c, a)
+		}
+		return nil
+	},
+	Short: "webhook starts a HTTPS server to validate updates and creations of Kilo peers.",
+	RunE:  webhook,
+}
+
+var (
+	certPath    string
+	keyPath     string
+	metricsAddr string
+	listenAddr  string
+)
+
+func init() {
+	webhookCmd.Flags().StringVar(&certPath, "cert-file", "", "The path to a certificate file")
+	webhookCmd.Flags().StringVar(&keyPath, "key-file", "", "The path to a key file")
+	webhookCmd.Flags().StringVar(&metricsAddr, "listen-metrics", ":1107", "The metrics server will be listening to that address")
+	webhookCmd.Flags().StringVar(&listenAddr, "listen", ":8443", "The webhook server will be listening to that address")
+}
+
+var deserializer = serializer.NewCodecFactory(runtime.NewScheme()).UniversalDeserializer()
+
+var (
+	validationCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "admission_requests_total",
+			Help: "The number of received admission reviews requests",
+		},
+		[]string{"operation", "response"},
+	)
+	requestCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_requests_total",
+			Help: "The number of received http requests",
+		},
+		[]string{"handler", "method"},
+	)
+	errorCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "errors_total",
+			Help: "The total number of errors",
+		},
+	)
+)
+
+func validationHandler(w http.ResponseWriter, r *http.Request) {
+	level.Debug(logger).Log("msg", "handling request", "source", r.RemoteAddr)
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		errorCounter.Inc()
+		level.Error(logger).Log("err", "failed to parse body from incoming request", "source", r.RemoteAddr)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	var admissionReview v1.AdmissionReview
+
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/json" {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("received Content-Type=%s, expected application/json", contentType)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+
+	response := v1.AdmissionReview{}
+
+	_, gvk, err := deserializer.Decode(body, nil, &admissionReview)
+	if err != nil {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("Request could not be decoded: %v", err)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+	if *gvk != v1.SchemeGroupVersion.WithKind("AdmissionReview") {
+		errorCounter.Inc()
+		msg := "only API v1 is supported"
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+	response.SetGroupVersionKind(*gvk)
+	response.Response = &v1.AdmissionResponse{
+		UID: admissionReview.Request.UID,
+	}
+
+	rawExtension := admissionReview.Request.Object
+	var peer kilo.Peer
+
+	if err := json.Unmarshal(rawExtension.Raw, &peer); err != nil {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("could not unmarshal extension to peer spec: %v:", err)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+
+	if err := peer.Validate(); err == nil {
+		level.Debug(logger).Log("msg", "got valid peer spec", "spec", peer.Spec, "name", peer.ObjectMeta.Name)
+		validationCounter.With(prometheus.Labels{"operation": string(admissionReview.Request.Operation), "response": "allowed"}).Inc()
+		response.Response.Allowed = true
+	} else {
+		level.Debug(logger).Log("msg", "got invalid peer spec", "spec", peer.Spec, "name", peer.ObjectMeta.Name)
+		validationCounter.With(prometheus.Labels{"operation": string(admissionReview.Request.Operation), "response": "denied"}).Inc()
+		response.Response.Result = &metav1.Status{
+			Message: err.Error(),
+		}
+	}
+
+	res, err := json.Marshal(response)
+	if err != nil {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("failed to marshal response: %v", err)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if _, err := w.Write(res); err != nil {
+		level.Error(logger).Log("err", err, "msg", "failed to write response")
+	}
+}
+
+func metricsMiddleWare(path string, next func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		requestCounter.With(prometheus.Labels{"method": r.Method, "handler": path}).Inc()
+		next(w, r)
+	}
+}
+
+func webhook(_ *cobra.Command, _ []string) error {
+	if printVersion {
+		fmt.Println(version.Version)
+		os.Exit(0)
+	}
+	registry.MustRegister(
+		errorCounter,
+		validationCounter,
+		requestCounter,
+	)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer func() {
+		cancel()
+	}()
+	var g run.Group
+	g.Add(run.SignalHandler(ctx, syscall.SIGINT, syscall.SIGTERM))
+	{
+		mm := http.NewServeMux()
+		mm.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+		msrv := &http.Server{
+			Addr:    metricsAddr,
+			Handler: mm,
+		}
+
+		g.Add(
+			func() error {
+				level.Info(logger).Log("msg", "starting metrics server", "address", msrv.Addr)
+				err := msrv.ListenAndServe()
+				level.Info(logger).Log("msg", "metrics server exited", "err", err)
+				return err
+
+			},
+			func(err error) {
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Info(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+				level.Info(logger).Log("msg", "shutting down metrics server gracefully")
+				ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+				defer func() {
+					cancel()
+				}()
+				if err := msrv.Shutdown(ctx); err != nil {
+					level.Error(logger).Log("msg", "failed to shut down metrics server gracefully", "err", err.Error())
+					msrv.Close()
+				}
+			},
+		)
+	}
+
+	{
+		mux := http.NewServeMux()
+		mux.HandleFunc("/validate", metricsMiddleWare("/validate", validationHandler))
+		srv := &http.Server{
+			Addr:    listenAddr,
+			Handler: mux,
+		}
+		g.Add(
+			func() error {
+				level.Info(logger).Log("msg", "starting webhook server", "address", srv.Addr)
+				err := srv.ListenAndServeTLS(certPath, keyPath)
+				level.Info(logger).Log("msg", "webhook server exited", "err", err)
+				return err
+			},
+			func(err error) {
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Info(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+				level.Info(logger).Log("msg", "shutting down webhook server gracefully")
+				ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+				defer func() {
+					cancel()
+				}()
+				if err := srv.Shutdown(ctx); err != nil {
+					level.Error(logger).Log("msg", "failed to shut down webhook server gracefully", "err", err.Error())
+					srv.Close()
+				}
+			},
+		)
+	}
+
+	err := g.Run()
+	var serr run.SignalError
+	if ok := errors.As(err, &serr); ok {
+		return nil
+	}
+	return err
+}

cmd/kgctl/connect_linux.go

@@ -0,0 +1,374 @@
+// Copyright 2022 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/oklog/run"
+	"github.com/spf13/cobra"
+	"golang.zx2c4.com/wireguard/wgctrl"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/squat/kilo/pkg/iproute"
+	"github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
+	"github.com/squat/kilo/pkg/mesh"
+	"github.com/squat/kilo/pkg/route"
+	"github.com/squat/kilo/pkg/wireguard"
+)
+
+var (
+	logLevel    string
+	connectOpts struct {
+		allowedIP           net.IPNet
+		allowedIPs          []net.IPNet
+		privateKey          string
+		cleanUp             bool
+		mtu                 uint
+		resyncPeriod        time.Duration
+		interfaceName       string
+		persistentKeepalive int
+	}
+)
+
+func takeIPNet(_ net.IP, i *net.IPNet, err error) *net.IPNet {
+	if err != nil {
+		panic(err)
+	}
+	return i
+}
+
+func connect() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:          "connect",
+		Args:         cobra.ExactArgs(1),
+		RunE:         runConnect,
+		Short:        "connect to a Kilo cluster as a peer over WireGuard",
+		SilenceUsage: true,
+	}
+	cmd.Flags().IPNetVarP(&connectOpts.allowedIP, "allowed-ip", "a", *takeIPNet(net.ParseCIDR("10.10.10.10/32")), "Allowed IP of the peer.")
+	cmd.Flags().StringSliceVar(&allowedIPs, "allowed-ips", []string{}, "Additional allowed IPs of the cluster, e.g. the service CIDR.")
+	cmd.Flags().StringVar(&logLevel, "log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels))
+	cmd.Flags().StringVar(&connectOpts.privateKey, "private-key", "", "Path to an existing WireGuard private key file.")
+	cmd.Flags().BoolVar(&connectOpts.cleanUp, "clean-up", true, "Should Kilo clean up the routes and interface when it shuts down?")
+	cmd.Flags().UintVar(&connectOpts.mtu, "mtu", uint(1420), "The MTU for the WireGuard interface.")
+	cmd.Flags().DurationVar(&connectOpts.resyncPeriod, "resync-period", 30*time.Second, "How often should Kilo reconcile?")
+	cmd.Flags().StringVarP(&connectOpts.interfaceName, "interface", "i", mesh.DefaultKiloInterface, "Name of the Kilo interface to use; if it does not exist, it will be created.")
+	cmd.Flags().IntVar(&connectOpts.persistentKeepalive, "persistent-keepalive", 10, "How often should WireGuard send keepalives? Setting to 0 will disable sending keepalives.")
+
+	availableLogLevels = strings.Join([]string{
+		logLevelAll,
+		logLevelDebug,
+		logLevelInfo,
+		logLevelWarn,
+		logLevelError,
+		logLevelNone,
+	}, ", ")
+
+	return cmd
+}
+
+func runConnect(cmd *cobra.Command, args []string) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	logger := log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
+	switch logLevel {
+	case logLevelAll:
+		logger = level.NewFilter(logger, level.AllowAll())
+	case logLevelDebug:
+		logger = level.NewFilter(logger, level.AllowDebug())
+	case logLevelInfo:
+		logger = level.NewFilter(logger, level.AllowInfo())
+	case logLevelWarn:
+		logger = level.NewFilter(logger, level.AllowWarn())
+	case logLevelError:
+		logger = level.NewFilter(logger, level.AllowError())
+	case logLevelNone:
+		logger = level.NewFilter(logger, level.AllowNone())
+	default:
+		return fmt.Errorf("log level %s unknown; possible values are: %s", logLevel, availableLogLevels)
+	}
+	logger = log.With(logger, "ts", log.DefaultTimestampUTC)
+	logger = log.With(logger, "caller", log.DefaultCaller)
+	peerName := args[0]
+
+	for i := range allowedIPs {
+		_, aip, err := net.ParseCIDR(allowedIPs[i])
+		if err != nil {
+			return err
+		}
+		connectOpts.allowedIPs = append(connectOpts.allowedIPs, *aip)
+	}
+
+	var privateKey wgtypes.Key
+	var err error
+	if connectOpts.privateKey == "" {
+		privateKey, err = wgtypes.GeneratePrivateKey()
+		if err != nil {
+			return fmt.Errorf("failed to generate private key: %w", err)
+		}
+	} else {
+		raw, err := os.ReadFile(connectOpts.privateKey)
+		if err != nil {
+			return fmt.Errorf("failed to read private key: %w", err)
+		}
+		privateKey, err = wgtypes.ParseKey(string(raw))
+		if err != nil {
+			return fmt.Errorf("failed to parse private key: %w", err)
+		}
+	}
+	publicKey := privateKey.PublicKey()
+	level.Info(logger).Log("msg", "generated public key", "key", publicKey)
+
+	if _, err := opts.kc.KiloV1alpha1().Peers().Get(ctx, peerName, metav1.GetOptions{}); apierrors.IsNotFound(err) {
+		peer := &v1alpha1.Peer{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: peerName,
+			},
+			Spec: v1alpha1.PeerSpec{
+				AllowedIPs:          []string{connectOpts.allowedIP.String()},
+				PersistentKeepalive: connectOpts.persistentKeepalive,
+				PublicKey:           publicKey.String(),
+			},
+		}
+		if _, err := opts.kc.KiloV1alpha1().Peers().Create(ctx, peer, metav1.CreateOptions{}); err != nil {
+			return fmt.Errorf("failed to create peer: %w", err)
+		}
+		level.Info(logger).Log("msg", "created peer", "peer", peerName)
+		if connectOpts.cleanUp {
+			defer func() {
+				ctxWithTimeout, cancelWithTimeout := context.WithTimeout(context.Background(), 10*time.Second)
+				defer cancelWithTimeout()
+				if err := opts.kc.KiloV1alpha1().Peers().Delete(ctxWithTimeout, peerName, metav1.DeleteOptions{}); err != nil {
+					level.Error(logger).Log("err", fmt.Sprintf("failed to delete peer: %v", err))
+				} else {
+					level.Info(logger).Log("msg", "deleted peer", "peer", peerName)
+				}
+			}()
+		}
+
+	} else if err != nil {
+		return fmt.Errorf("failed to get peer: %w", err)
+	}
+
+	iface, _, err := wireguard.New(connectOpts.interfaceName, connectOpts.mtu)
+	if err != nil {
+		return fmt.Errorf("failed to create wg interface: %w", err)
+	}
+	level.Info(logger).Log("msg", "created WireGuard interface", "name", connectOpts.interfaceName, "index", iface)
+
+	table := route.NewTable()
+	if connectOpts.cleanUp {
+		defer cleanUp(iface, table, logger)
+	}
+
+	if err := iproute.SetAddress(iface, &connectOpts.allowedIP); err != nil {
+		return err
+	}
+	level.Info(logger).Log("msg", "set IP address of WireGuard interface", "IP", connectOpts.allowedIP.String())
+
+	if err := iproute.Set(iface, true); err != nil {
+		return err
+	}
+
+	var g run.Group
+	g.Add(run.SignalHandler(ctx, syscall.SIGINT, syscall.SIGTERM))
+
+	{
+		g.Add(
+			func() error {
+				errCh, err := table.Run(ctx.Done())
+				if err != nil {
+					return fmt.Errorf("failed to watch for route table updates: %w", err)
+				}
+				for {
+					select {
+					case err, ok := <-errCh:
+						if ok {
+							level.Error(logger).Log("err", err.Error())
+						} else {
+							return nil
+						}
+					case <-ctx.Done():
+						return nil
+					}
+				}
+			},
+			func(err error) {
+				cancel()
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Debug(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+			},
+		)
+	}
+	{
+		g.Add(
+			func() error {
+				level.Info(logger).Log("msg", "starting syncer")
+				for {
+					if err := sync(table, peerName, privateKey, iface, logger); err != nil {
+						level.Error(logger).Log("msg", "failed to sync", "err", err.Error())
+					}
+					select {
+					case <-time.After(connectOpts.resyncPeriod):
+					case <-ctx.Done():
+						return nil
+					}
+				}
+			}, func(err error) {
+				cancel()
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Debug(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+			})
+	}
+
+	err = g.Run()
+	var serr run.SignalError
+	if ok := errors.As(err, &serr); ok {
+		return nil
+	}
+	return err
+}
+
+func cleanUp(iface int, t *route.Table, logger log.Logger) {
+	if err := iproute.Set(iface, false); err != nil {
+		level.Error(logger).Log("err", fmt.Sprintf("failed to set WireGuard interface down: %v", err))
+	}
+	if err := iproute.RemoveInterface(iface); err != nil {
+		level.Error(logger).Log("err", fmt.Sprintf("failed to remove WireGuard interface: %v", err))
+	}
+	if err := t.CleanUp(); err != nil {
+		level.Error(logger).Log("failed to clean up routes: %v", err)
+	}
+
+	return
+}
+
+func sync(table *route.Table, peerName string, privateKey wgtypes.Key, iface int, logger log.Logger) error {
+	ns, err := opts.backend.Nodes().List()
+	if err != nil {
+		return fmt.Errorf("failed to list nodes: %w", err)
+	}
+	for _, n := range ns {
+		_, err := n.Endpoint.UDPAddr(true)
+		if err != nil {
+			return err
+		}
+	}
+	ps, err := opts.backend.Peers().List()
+	if err != nil {
+		return fmt.Errorf("failed to list peers: %w", err)
+	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
+	}
+	var hostname string
+	var subnet *net.IPNet
+	nodes := make(map[string]*mesh.Node)
+	var nodeNames []string
+	for _, n := range ns {
+		if n.Ready() {
+			nodes[n.Name] = n
+			hostname = n.Name
+			nodeNames = append(nodeNames, n.Name)
+		}
+		if n.WireGuardIP != nil && subnet == nil {
+			subnet = n.WireGuardIP
+		}
+	}
+	if len(nodes) == 0 {
+		return errors.New("did not find any valid Kilo nodes in the cluster")
+	}
+	if subnet == nil {
+		return errors.New("did not find a valid Kilo subnet on any node")
+	}
+	subnet.IP = subnet.IP.Mask(subnet.Mask)
+	sort.Strings(nodeNames)
+	nodes[nodeNames[0]].AllowedLocationIPs = append(nodes[nodeNames[0]].AllowedLocationIPs, connectOpts.allowedIPs...)
+	peers := make(map[string]*mesh.Peer)
+	for _, p := range ps {
+		if p.Ready() {
+			peers[p.Name] = p
+		}
+	}
+	if _, ok := peers[peerName]; !ok {
+		return fmt.Errorf("did not find any peer named %q in the cluster", peerName)
+	}
+
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, wgtypes.Key{}, subnet, *peers[peerName].PersistentKeepaliveInterval, logger)
+	if err != nil {
+		return fmt.Errorf("failed to create topology: %w", err)
+	}
+	conf := t.PeerConf(peerName)
+	conf.PrivateKey = &privateKey
+	conf.ListenPort = &opts.port
+
+	wgClient, err := wgctrl.New()
+	if err != nil {
+		return err
+	}
+	defer wgClient.Close()
+
+	current, err := wgClient.Device(connectOpts.interfaceName)
+	if err != nil {
+		return err
+	}
+
+	var equal bool
+	var diff string
+	equal, diff = conf.Equal(current)
+	if !equal {
+		// If the key is empty, then it's the first time we are running
+		// so don't bother printing a diff.
+		if current.PrivateKey != [wgtypes.KeyLen]byte{} {
+			level.Info(logger).Log("msg", "WireGuard configurations are different", "diff", diff)
+		}
+		level.Debug(logger).Log("msg", "setting WireGuard config", "config", conf.WGConfig())
+		if err := wgClient.ConfigureDevice(connectOpts.interfaceName, conf.WGConfig()); err != nil {
+			return err
+		}
+	}
+
+	if err := table.Set(t.PeerRoutes(peerName, iface, connectOpts.allowedIPs)); err != nil {
+		return fmt.Errorf("failed to update route table: %w", err)
+	}
+
+	return nil
+}

cmd/kgctl/connect_other.go

@@ -1,10 +1,10 @@
-// Copyright 2015 go-swagger maintainers
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//    http://www.apache.org/licenses/LICENSE-2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,13 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package spec
+//go:build !linux
+// +build !linux
 
-// ExternalDocumentation allows referencing an external resource for
-// extended documentation.
-//
-// For more information: http://goo.gl/8us55a#externalDocumentationObject
-type ExternalDocumentation struct {
-	Description string `json:"description,omitempty"`
-	URL         string `json:"url,omitempty"`
+package main
+
+import (
+	"errors"
+
+	"github.com/spf13/cobra"
+)
+
+func connect() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "connect",
+		Short: "not supporred on this OS",
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return errors.New("this command is not supported on this OS")
+		},
+	}
+	return cmd
 }

cmd/kgctl/graph.go

@@ -18,6 +18,8 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
 	"github.com/squat/kilo/pkg/mesh"
 )
 
@@ -32,12 +34,17 @@ func graph() *cobra.Command {
 func runGraph(_ *cobra.Command, _ []string) error {
 	ns, err := opts.backend.Nodes().List()
 	if err != nil {
-		return fmt.Errorf("failed to list nodes: %v", err)
+		return fmt.Errorf("failed to list nodes: %w", err)
 	}
 	ps, err := opts.backend.Peers().List()
 	if err != nil {
-		return fmt.Errorf("failed to list peers: %v", err)
+		return fmt.Errorf("failed to list peers: %w", err)
 	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
+	}
+
 	var hostname string
 	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
@@ -60,13 +67,13 @@ func runGraph(_ *cobra.Command, _ []string) error {
 			peers[p.Name] = p
 		}
 	}
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, wgtypes.Key{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create topology: %v", err)
+		return fmt.Errorf("failed to create topology: %w", err)
 	}
 	g, err := t.Dot()
 	if err != nil {
-		return fmt.Errorf("failed to generate graph: %v", err)
+		return fmt.Errorf("failed to generate graph: %w", err)
 	}
 	fmt.Println(g)
 	return nil

cmd/kgctl/main.go

@@ -15,10 +15,13 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
+	"github.com/go-kit/kit/log"
 	"github.com/spf13/cobra"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
@@ -46,6 +49,7 @@ var (
 	availableGranularities = strings.Join([]string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.AutoGranularity),
 	}, ", ")
 	availableLogLevels = strings.Join([]string{
 		logLevelAll,
@@ -58,7 +62,8 @@ var (
 	opts struct {
 		backend     mesh.Backend
 		granularity mesh.Granularity
-		port        uint32
+		kc          kiloclient.Interface
+		port        int
 	}
 	backend       string
 	granularity   string
@@ -67,34 +72,39 @@ var (
 )
 
 func runRoot(_ *cobra.Command, _ []string) error {
+	if opts.port < 1 || opts.port > 1<<16-1 {
+		return fmt.Errorf("invalid port: port mus be in range [%d:%d], but got %d", 1, 1<<16-1, opts.port)
+	}
+
 	opts.granularity = mesh.Granularity(granularity)
 	switch opts.granularity {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.AutoGranularity:
 	default:
-		return fmt.Errorf("mesh granularity %v unknown; posible values are: %s", granularity, availableGranularities)
+		return fmt.Errorf("mesh granularity %s unknown; posible values are: %s", granularity, availableGranularities)
 	}
 
 	switch backend {
 	case k8s.Backend:
 		config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 		if err != nil {
-			return fmt.Errorf("failed to create Kubernetes config: %v", err)
+			return fmt.Errorf("failed to create Kubernetes config: %w", err)
 		}
 		c := kubernetes.NewForConfigOrDie(config)
-		kc := kiloclient.NewForConfigOrDie(config)
+		opts.kc = kiloclient.NewForConfigOrDie(config)
 		ec := apiextensions.NewForConfigOrDie(config)
-		opts.backend = k8s.New(c, kc, ec, topologyLabel)
+		opts.backend = k8s.New(c, opts.kc, ec, topologyLabel, log.NewNopLogger())
 	default:
-		return fmt.Errorf("backend %v unknown; posible values are: %s", backend, availableBackends)
+		return fmt.Errorf("backend %s unknown; posible values are: %s", backend, availableBackends)
 	}
 
 	if err := opts.backend.Nodes().Init(make(chan struct{})); err != nil {
-		return fmt.Errorf("failed to initialize node backend: %v", err)
+		return fmt.Errorf("failed to initialize node backend: %w", err)
 	}
 
 	if err := opts.backend.Peers().Init(make(chan struct{})); err != nil {
-		return fmt.Errorf("failed to initialize peer backend: %v", err)
+		return fmt.Errorf("failed to initialize peer backend: %w", err)
 	}
 	return nil
 }
@@ -106,16 +116,22 @@ func main() {
 		Long:              "",
 		PersistentPreRunE: runRoot,
 		Version:           version.Version,
+		SilenceErrors:     true,
 	}
 	cmd.PersistentFlags().StringVar(&backend, "backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
-	cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
-	cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig.")
-	cmd.PersistentFlags().Uint32Var(&opts.port, "port", mesh.DefaultKiloPort, "The WireGuard port over which the nodes communicate.")
+	cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.AutoGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
+	defaultKubeconfig := os.Getenv("KUBECONFIG")
+	if _, err := os.Stat(defaultKubeconfig); os.IsNotExist(err) {
+		defaultKubeconfig = filepath.Join(os.Getenv("HOME"), ".kube/config")
+	}
+	cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", defaultKubeconfig, "Path to kubeconfig.")
+	cmd.PersistentFlags().IntVar(&opts.port, "port", mesh.DefaultKiloPort, "The WireGuard port over which the nodes communicate.")
 	cmd.PersistentFlags().StringVar(&topologyLabel, "topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
 
 	for _, subCmd := range []*cobra.Command{
 		graph(),
 		showConf(),
+		connect(),
 	} {
 		cmd.AddCommand(subCmd)
 	}
@@ -125,3 +141,20 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func determineGranularity(gr mesh.Granularity, ns []*mesh.Node) (mesh.Granularity, error) {
+	if gr == mesh.AutoGranularity {
+		if len(ns) == 0 {
+			return gr, errors.New("could not get any nodes")
+		}
+		ret := mesh.Granularity(ns[0].Granularity)
+		switch ret {
+		case mesh.LogicalGranularity:
+		case mesh.FullGranularity:
+		default:
+			return ret, fmt.Errorf("mesh granularity %s is not supported", opts.granularity)
+		}
+		return ret, nil
+	}
+	return gr, nil
+}

cmd/kgctl/showconf.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,14 +15,15 @@
 package main
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"net"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -47,7 +48,7 @@ var (
 	}, ", ")
 	allowedIPs   []string
 	showConfOpts struct {
-		allowedIPs []*net.IPNet
+		allowedIPs []net.IPNet
 		serializer *json.Serializer
 		output     string
 		asPeer     bool
@@ -82,14 +83,14 @@ func runShowConf(c *cobra.Command, args []string) error {
 	case outputFormatYAML:
 		showConfOpts.serializer = json.NewYAMLSerializer(json.DefaultMetaFactory, peerCreatorTyper{}, peerCreatorTyper{})
 	default:
-		return fmt.Errorf("output format %v unknown; posible values are: %s", showConfOpts.output, availableOutputFormats)
+		return fmt.Errorf("output format %s unknown; posible values are: %s", showConfOpts.output, availableOutputFormats)
 	}
 	for i := range allowedIPs {
 		_, aip, err := net.ParseCIDR(allowedIPs[i])
 		if err != nil {
 			return fmt.Errorf("allowed-ips must contain only valid CIDRs; got %q", allowedIPs[i])
 		}
-		showConfOpts.allowedIPs = append(showConfOpts.allowedIPs, aip)
+		showConfOpts.allowedIPs = append(showConfOpts.allowedIPs, *aip)
 	}
 	return runRoot(c, args)
 }
@@ -115,11 +116,15 @@ func showConfPeer() *cobra.Command {
 func runShowConfNode(_ *cobra.Command, args []string) error {
 	ns, err := opts.backend.Nodes().List()
 	if err != nil {
-		return fmt.Errorf("failed to list nodes: %v", err)
+		return fmt.Errorf("failed to list nodes: %w", err)
 	}
 	ps, err := opts.backend.Peers().List()
 	if err != nil {
-		return fmt.Errorf("failed to list peers: %v", err)
+		return fmt.Errorf("failed to list peers: %w", err)
+	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
 	}
 	hostname := args[0]
 	subnet := mesh.DefaultKiloSubnet
@@ -147,14 +152,14 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		}
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, int(opts.port), wgtypes.Key{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create topology: %v", err)
+		return fmt.Errorf("failed to create topology: %w", err)
 	}
 
 	var found bool
 	for _, p := range t.PeerConf("").Peers {
-		if bytes.Equal(p.PublicKey, nodes[hostname].Key) {
+		if p.PublicKey == nodes[hostname].Key {
 			found = true
 			break
 		}
@@ -167,7 +172,7 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 	if !showConfOpts.asPeer {
 		c, err := t.Conf().Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -178,6 +183,9 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		fallthrough
 	case outputFormatYAML:
 		p := t.AsPeer()
+		if p == nil {
+			return errors.New("cannot generate config from nil peer")
+		}
 		p.AllowedIPs = append(p.AllowedIPs, showConfOpts.allowedIPs...)
 		p.DeduplicateIPs()
 		k8sp := translatePeer(p)
@@ -185,13 +193,16 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		return showConfOpts.serializer.Encode(k8sp, os.Stdout)
 	case outputFormatWireGuard:
 		p := t.AsPeer()
+		if p == nil {
+			return errors.New("cannot generate config from nil peer")
+		}
 		p.AllowedIPs = append(p.AllowedIPs, showConfOpts.allowedIPs...)
 		p.DeduplicateIPs()
 		c, err := (&wireguard.Conf{
-			Peers: []*wireguard.Peer{p},
+			Peers: []wireguard.Peer{*p},
 		}).Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -202,11 +213,15 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 func runShowConfPeer(_ *cobra.Command, args []string) error {
 	ns, err := opts.backend.Nodes().List()
 	if err != nil {
-		return fmt.Errorf("failed to list nodes: %v", err)
+		return fmt.Errorf("failed to list nodes: %w", err)
 	}
 	ps, err := opts.backend.Peers().List()
 	if err != nil {
-		return fmt.Errorf("failed to list peers: %v", err)
+		return fmt.Errorf("failed to list peers: %w", err)
+	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
 	}
 	var hostname string
 	subnet := mesh.DefaultKiloSubnet
@@ -236,14 +251,18 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		return fmt.Errorf("did not find any peer named %q in the cluster", peer)
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet, peers[peer].PersistentKeepalive)
+	pka := time.Duration(0)
+	if p := peers[peer].PersistentKeepaliveInterval; p != nil {
+		pka = *p
+	}
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, wgtypes.Key{}, subnet, pka, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create topology: %v", err)
+		return fmt.Errorf("failed to create topology: %w", err)
 	}
 	if !showConfOpts.asPeer {
 		c, err := t.PeerConf(peer).Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -264,10 +283,10 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		p.AllowedIPs = append(p.AllowedIPs, showConfOpts.allowedIPs...)
 		p.DeduplicateIPs()
 		c, err := (&wireguard.Conf{
-			Peers: []*wireguard.Peer{p},
+			Peers: []wireguard.Peer{*p},
 		}).Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -276,6 +295,7 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 }
 
 // translatePeer translates a wireguard.Peer to a Peer CRD.
+// TODO this function has many similarities to peerBackend.Set(name, peer)
 func translatePeer(peer *wireguard.Peer) *v1alpha1.Peer {
 	if peer == nil {
 		return &v1alpha1.Peer{}
@@ -283,36 +303,33 @@ func translatePeer(peer *wireguard.Peer) *v1alpha1.Peer {
 	var aips []string
 	for _, aip := range peer.AllowedIPs {
 		// Skip any invalid IPs.
-		if aip == nil {
+		// TODO all IPs should be valid, so no need to skip here?
+		if aip.String() == (&net.IPNet{}).String() {
 			continue
 		}
 		aips = append(aips, aip.String())
 	}
 	var endpoint *v1alpha1.PeerEndpoint
-	if peer.Endpoint != nil && peer.Endpoint.Port > 0 && (peer.Endpoint.IP != nil || peer.Endpoint.DNS != "") {
-		var ip string
-		if peer.Endpoint.IP != nil {
-			ip = peer.Endpoint.IP.String()
-		}
+	if peer.Endpoint.Port() > 0 || !peer.Endpoint.HasDNS() {
 		endpoint = &v1alpha1.PeerEndpoint{
 			DNSOrIP: v1alpha1.DNSOrIP{
-				DNS: peer.Endpoint.DNS,
-				IP:  ip,
+				IP:  peer.Endpoint.IP().String(),
+				DNS: peer.Endpoint.DNS(),
 			},
-			Port: peer.Endpoint.Port,
+			Port: uint32(peer.Endpoint.Port()),
 		}
 	}
 	var key string
-	if len(peer.PublicKey) > 0 {
-		key = string(peer.PublicKey)
+	if peer.PublicKey != (wgtypes.Key{}) {
+		key = peer.PublicKey.String()
 	}
 	var psk string
-	if len(peer.PresharedKey) > 0 {
-		psk = string(peer.PresharedKey)
+	if peer.PresharedKey != nil {
+		psk = peer.PresharedKey.String()
 	}
 	var pka int
-	if peer.PersistentKeepalive > 0 {
-		pka = peer.PersistentKeepalive
+	if peer.PersistentKeepaliveInterval != nil && *peer.PersistentKeepaliveInterval > time.Duration(0) {
+		pka = int(*peer.PersistentKeepaliveInterval)
 	}
 	return &v1alpha1.Peer{
 		TypeMeta: metav1.TypeMeta{

docs/annotations.md

@@ -9,6 +9,7 @@ The following annotations can be added to any Kubernetes Node object to configur
 |[kilo.squat.ai/leader](#leader)|string|`""`, `true`|
 |[kilo.squat.ai/location](#location)|string|`gcp-east`, `lab`|
 |[kilo.squat.ai/persistent-keepalive](#persistent-keepalive)|uint|`10`|
+|[kilo.squat.ai/allowed-location-ips](#allowed-location-ips)|CIDR|`66.66.66.66/32`|
 
 ### force-endpoint
 In order to create links between locations, Kilo requires at least one node in each location to have an endpoint, ie a `host:port` combination, that is routable from the other locations.
@@ -35,7 +36,7 @@ In some situations it may be desirable to manually select the leader for a locat
  * _firewall_: Kilo requires an open UDP port, which defaults to 51820, to communicate between locations; if only one node is configured to have that port open, then that node should be given the leader annotation;
  * _bandwidth_: if certain nodes in the cluster have a higher bandwidth or lower latency Internet connection, then those nodes should be given the leader annotation.
 
-_Note_: multiple nodes within a single location can be given the leader annotation; in this case, Kilo will select one leader from the set of annotated nodes. 
+> **Note**: multiple nodes within a single location can be given the leader annotation; in this case, Kilo will select one leader from the set of annotated nodes.
 
 ### location
 Kilo allows nodes in different logical or physical locations to route packets to one-another.
@@ -43,7 +44,7 @@ In order to know what connections to create, Kilo needs to know which nodes are
 Kilo will try to infer each node's location from the [topology.kubernetes.io/region](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#topologykubernetesioregion) node label.
 If the label is not present for a node, for example if running a bare-metal cluster or on an unsupported cloud provider, then the location annotation should be specified.
 
-_Note_: all nodes without a defined location will be considered to be in the default location `""`.
+> **Note**: all nodes without a defined location will be considered to be in the default location `""`.
 
 ### persistent-keepalive
 In certain deployments, cluster nodes may be located behind NAT or a firewall, e.g. edge nodes located behind a commodity router.
@@ -52,3 +53,10 @@ In order for a node behind NAT to receive packets from nodes outside of the NATe
 The frequency of emission of these keepalive packets can be controlled by setting the persistent-keepalive annotation on the node behind NAT.
 The annotated node will use the specified value will as the persistent-keepalive interval for all of its peers.
 For more background, [see the WireGuard documentation on NAT and firewall traversal](https://www.wireguard.com/quickstart/#nat-and-firewall-traversal-persistence).
+
+### allowed-location-ips
+It is possible to add allowed-location-ips to a location by annotating any node within that location.
+Adding allowed-location-ips to a location makes these IPs routable from other locations as well.
+
+In an example deployment of Kilo with two locations A and B, a printer in location A can be accessible from nodes and pods in location B.
+Additionally, Kilo Peers can use the printer in location A.

docs/api.md

@@ -0,0 +1,69 @@
+# API
+This document is a reference of the API types introduced by Kilo.
+
+> **Note**: this document is generated from code comments. When contributing a change to this document, please do so by changing the code comments.
+
+## Table of Contents
+* [DNSOrIP](#dnsorip)
+* [Peer](#peer)
+* [PeerEndpoint](#peerendpoint)
+* [PeerList](#peerlist)
+* [PeerSpec](#peerspec)
+
+## DNSOrIP
+
+DNSOrIP represents either a DNS name or an IP address. When both are given, the IP address, as it is more specific, override the DNS name.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| dns | DNS must be a valid RFC 1123 subdomain. | string | false |
+| ip | IP must be a valid IP address. | string | false |
+
+[Back to TOC](#table-of-contents)
+
+## Peer
+
+Peer is a WireGuard peer that should have access to the VPN.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| metadata | Standard object’s metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#metadata | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#objectmeta-v1-meta) | false |
+| spec | Specification of the desired behavior of the Kilo Peer. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status | [PeerSpec](#peerspec) | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerEndpoint
+
+PeerEndpoint represents a WireGuard endpoint, which is an IP:port tuple.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| dnsOrIP | DNSOrIP is a DNS name or an IP address. | [DNSOrIP](#dnsorip) | true |
+| port | Port must be a valid port number. | uint32 | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerList
+
+PeerList is a list of peers.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| metadata | Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | [metav1.ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#listmeta-v1-meta) | false |
+| items | List of peers. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md | [][Peer](#peer) | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerSpec
+
+PeerSpec is the description and configuration of a peer.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| allowedIPs | AllowedIPs is the list of IP addresses that are allowed for the given peer's tunnel. | []string | true |
+| endpoint | Endpoint is the initial endpoint for connections to the peer. | *[PeerEndpoint](#peerendpoint) | false |
+| persistentKeepalive | PersistentKeepalive is the interval in seconds of the emission of keepalive packets by the peer. This defaults to 0, which disables the feature. | int | false |
+| presharedKey | PresharedKey is the optional symmetric encryption key for the peer. | string | false |
+| publicKey | PublicKey is the WireGuard public key for the peer. | string | true |
+
+[Back to TOC](#table-of-contents)

docs/building_kilo.md

@@ -0,0 +1,100 @@
+# Build and Test Kilo
+
+This document describes how you can build and test Kilo.
+
+To follow along, you need to install the following utilities:
+ - `go` not for building but formatting the code and running unit tests
+ - `make`
+ - `jq`
+ - `git`
+ - `curl`
+ - `docker`
+
+## Getting Started
+
+Clone the Repository and `cd` into it.
+```shell
+git clone https://github.com/squat/kilo.git
+cd kilo
+```
+
+## Build
+
+For consistency, the Kilo binaries are compiled in a Docker container, so make sure the `docker` package is installed and the daemon is running.
+
+### Compile Binaries
+
+To compile the `kg` and `kgctl` binaries run:
+```shell
+make
+```
+Binaries are always placed in a directory corresponding to the local system's OS and architecture following the pattern `bin/<os>/<architecture>/`, so on an AMD64 machine running Linux, the binaries will be stored in `bin/linux/amd64/`.
+
+You can build the binaries for a different architecture by setting the `ARCH` environment variable before invoking `make`, e.g.:
+```shell
+ARCH=<arm|arm64|amd64> make
+```
+
+Likewise, to build `kg` for another OS, set the `OS` environment variable before invoking `make`:
+```shell
+OS=<windows|darwin|linux> make
+```
+## Test
+
+To execute the unit tests, run:
+```shell
+make unit
+```
+
+To lint the code in the repository, run:
+```shell
+make lint
+```
+
+To execute basic end to end tests, run:
+```shell
+make e2e
+```
+> **Note**: The end to end tests are currently flaky, so try running them again if they fail.
+
+To instead run all of the tests with a single command, run:
+```shell
+make test
+```
+
+## Build and Push the Container Images
+
+If you want to build containers for a processor architecture that is different from your computer's, then you will first need to configure QEMU as the interpreter for binaries built for non-native architectures:
+```shell
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+
+Set the `$IMAGE` environment variable to `<your Docker Hub user name>/kilo`.
+This way the generated container images and manifests will be named accordingly.
+By skipping this step, you will be able to tag images but will not be able to push the containers and manifests to your own Docker Hub.
+```shell
+export IMAGE=<docker hub user name>/kilo
+```
+
+If you want to use a different container registry, run:
+```shell
+export REGISTRY=<your registry without a trailing slash>
+```
+
+To build containers with the `kg` image for `arm`, `arm64` and `amd64`, run:
+```shell
+make all-container
+```
+
+Push the container images and build a manifest with:
+```shell
+make manifest
+```
+
+To tag and push the manifest with `latest`, run:
+```shell
+make manifest-latest
+```
+
+Now you can deploy the custom build of Kilo to your cluster.
+If you are already running Kilo, change the image from `squat/kilo` to `[registry/]<username>/kilo[:sha]`.

docs/building_website.md

@@ -0,0 +1,44 @@
+# Build and Test the Website
+
+You may have noticed that the `markdown` files in the `/docs` directory are also displayed on [Kilo's website](https://kilo.squat.ai/).
+If you want to add documentation to Kilo, you can start a local webserver to check out how the website would look like.
+
+## Requirements
+
+Install [yarn](https://yarnpkg.com/getting-started/install).
+
+## Build and Run
+
+The markdown files for the website are located in `/website/docs` and are generated from the like-named markdown files in the `/docs` directory and from the corresponding header files without the `.md` extension in the `/website/docs` directory.
+To generate the markdown files in `/website/docs`, run:
+```shell
+make website/docs/README.md
+```
+
+Next, build the website itself by installing the `node_modules` and building the website's HTML from the generated markdown:
+```shell
+make website/build/index.html
+```
+
+Now, start the website server with:
+```shell
+yarn --cwd website start
+```
+This command should have opened a browser window with the website; if not, open your browser and point it to `http://localhost:3000`.
+
+If you make changes to any of the markdown files in `/docs` and want to reload the local `node` server, run:
+```shell
+make website/docs/README.md -B
+```
+
+You can execute the above while the node server is running and the website will be rebuilt and reloaded automatically.
+
+## Add a New File to the Docs
+
+If you add a new file to the `/docs` directory, you also need to create a corresponding header file containing the front-matter in `/website/docs/`.
+Then, regenerate the markdown for the website with the command:
+```shell
+make website/docs/README.md
+```
+Edit `/website/sidebars.js` accordingly.
+> **Note**: The `id` in the header file `/website/docs/<new file>` must match the `id` specified in `website/sidebars.js`.

docs/grafana/kilo.json

@@ -0,0 +1,962 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.5.4"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 12,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (pod) (rate(wireguard_received_bytes_total[1h])) + sum by (pod) (rate(wireguard_sent_bytes_total[1h]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [
+        {
+          "$$hashKey": "object:64",
+          "colorMode": "background6",
+          "fill": true,
+          "fillColor": "rgba(234, 112, 112, 0.12)",
+          "line": false,
+          "lineColor": "rgba(237, 46, 24, 0.60)",
+          "op": "time"
+        }
+      ],
+      "timeShift": null,
+      "title": "Throughput",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:42",
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:43",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "(sum(rate(wireguard_sent_bytes_total[5m])) - sum(rate(wireguard_received_bytes_total[5m])))/(sum(rate(wireguard_sent_bytes_total[5m])) + sum(rate(wireguard_received_bytes_total[5m])))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Slip (send - received)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:502",
+          "format": "percentunit",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:503",
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 16,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum by (public_key) (time() - (wireguard_latest_handshake_seconds!=0))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "latest handshake",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:219",
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": "1000",
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:220",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 18,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (instance) (rate(kilo_reconciles_total[30m]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "kilo reconciles",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:539",
+          "decimals": null,
+          "format": "hertz",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:540",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 0,
+        "y": 16
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.5.4",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "avg(kilo_peers)",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "title": "Kilo Peers",
+      "type": "stat"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 4,
+        "y": 16
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.5.4",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "avg(kilo_nodes)",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Kilo Nodes",
+      "type": "stat"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 8,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.5.4",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum(kilo_leader)",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "title": "segments",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum by (instance) (rate(kilo_errors_total[10m]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Kilo Errors",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:446",
+          "format": "hertz",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:447",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "hiddenSeries": false,
+      "id": 20,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (instance) (rate(process_cpu_seconds_total{pod=~\"kilo-.*\"}[1m]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "CPU usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:162",
+          "format": "percentunit",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:163",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "hiddenSeries": false,
+      "id": 22,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum by (instance) (process_resident_memory_bytes{pod=~\"kilo-.*\"})",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Memory Allocation",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:231",
+          "format": "decbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:232",
+          "format": "decmbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 32
+      },
+      "id": 14,
+      "panels": [],
+      "title": "Row title",
+      "type": "row"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Kilo",
+  "uid": "R8Lja3H7z",
+  "version": 11
+}

docs/kg.md

@@ -32,6 +32,7 @@ Usage of bin//linux/amd64/kg:
       --log-level string          Log level to use. Possible values: all, debug, info, warn, error, none (default "info")
       --master string             The address of the Kubernetes API server (overrides any value in kubeconfig).
       --mesh-granularity string   The granularity of the network mesh to create. Possible values: location, full (default "location")
+      --mtu uint                  The MTU of the WireGuard interface created by Kilo. (default 1420)
       --port uint                 The port over which WireGuard peers should communicate. (default 51820)
       --resync-period duration    How often should the Kilo controllers reconcile? (default 30s)
       --subnet string             CIDR from which to allocate addresses for WireGuard interfaces. (default "10.4.0.0/16")

docs/kgctl.md

@@ -6,21 +6,95 @@ This tool can be used to understand a mesh's topology, get the WireGuard configu
 
 ## Installation
 
-Installing `kgctl` currently requires building the binary from source.
-*Note*: the [Go toolchain must be installed](https://golang.org/doc/install) in order to build the binary.
-To build and install `kgctl`, run:
+The `kgctl` binary is automatically compiled for Linux, macOS, and Windows for every release of Kilo and can be downloaded from [the GitHub releases page](https://github.com/squat/kilo/releases/latest).
+
+### Building from Source
+Kilo is written in Golang and as a result the [Go toolchain must be installed](https://golang.org/doc/install) in order to build the `kgctl` binary.
+To download the Kilo source code and then build and install `kgctl` using the latest commit all with a single command, run:
 
 ```shell
-go install github.com/squat/kilo/cmd/kgctl
+go install github.com/squat/kilo/cmd/kgctl@latest
+```
+
+Alternatively, `kgctl` can be built and installed based on specific version of the code by specifying a Git tag or hash, e.g.:
+
+```shell
+go install github.com/squat/kilo/cmd/kgctl@0.2.0
+```
+
+When working on Kilo locally, it can be helpful to build and test the `kgctl` binary as part of the development cycle.
+In order to build a binary from a local checkout of the Git repository, run:
+
+```shell
+make
+```
+
+This will produce a `kgctl` binary at `./bin/<your-os>/<your-architecture>/kgctl`.
+
+
+### Binary Packages
+
+#### Arch Linux
+
+Install `kgctl` from the Arch User Repository using an AUR helper like `paru` or `yay`:
+
+```shell
+paru -S kgctl-bin
+```
+
+#### Arkade
+
+The [arkade](https://github.com/alexellis/arkade) CLI can be used to install `kgctl` on any OS and architecture:
+
+```shell
+arkade get kgctl
 ```
 
 ## Commands
 
 |Command|Syntax|Description|
 |----|----|-------|
+|[connect](#connect)|`kgctl connect <peer-name> [flags]`|Connect the host to the cluster, setting up the required interfaces, routes, and keys.|
 |[graph](#graph)|`kgctl graph [flags]`|Produce a graph in GraphViz format representing the topology of the cluster.|
-|[showconf](#showconf)|`kgctl showconf ( node \| peer ) NAME [flags]`|Show the WireGuard configuration for a node or peer in the mesh.|
+|[showconf](#showconf)|`kgctl showconf ( node \| peer ) <name> [flags]`|Show the WireGuard configuration for a node or peer in the mesh.|
 
+### connect
+
+The `connect` command configures the local host as a WireGuard Peer of the cluster and applies all of the necessary networking configuration to connect to the cluster.
+As long as the process is running, it will watch the cluster for changes and automatically manage the configuration for new or updated Peers and Nodes.
+If the given Peer name does not exist in the cluster, the command will register a new Peer and generate the necessary WireGuard keys.
+When the command exits, all of the configuration, including newly registered Peers, is cleaned up.
+
+Example:
+
+```shell
+PEER_NAME=laptop
+SERVICECIDR=10.43.0.0/16
+kgctl connect $PEER_NAME --allowed-ips $SERVICECIDR
+```
+
+The local host is now connected to the cluster and all IPs from the cluster and any registered Peers are fully routable.
+When combined with the `--clean-up false` flag, the configuration produced by the command is persistent and will remain in effect even after the process is stopped.
+
+With the service CIDR of the cluster routable from the local host, Kubernetes DNS names can now be resolved by the cluster DNS provider.
+For example, the following snippet could be used to resolve the clusterIP of the Kubernetes API:
+```shell
+dig @$(kubectl get service -n kube-system kube-dns -o=jsonpath='{.spec.clusterIP}') kubernetes.default.svc.cluster.local +short
+# > 10.43.0.1
+```
+
+For convenience, the cluster DNS provider's IP address can be configured as the local host's DNS server, making Kubernetes DNS names easily resolvable.
+For example, if using `systemd-resolved`, the following snippet could be used:
+```shell
+systemd-resolve --interface kilo0 --set-dns $(kubectl get service -n kube-system kube-dns -o=jsonpath='{.spec.clusterIP}') --set-domain cluster.local
+# Now all lookups for DNS names ending in `.cluster.local` will be routed over the `kilo0` interface to the cluster DNS provider.
+dig kubernetes.default.svc.cluster.local +short
+# > 10.43.0.1
+```
+
+> **Note**: The `connect` command is currently only supported on Linux.
+
+> **Note**: The `connect` command requires the `CAP_NET_ADMIN` capability in order to configure the host's networking stack; unprivileged users will need to use `sudo` or similar tools.
 
 ### graph
 

docs/monitoring.md

@@ -0,0 +1,100 @@
+# Monitoring
+
+The following assumes that you have applied the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) monitoring stack onto your cluster.
+
+## Kilo
+
+Monitor the Kilo DaemonSet with:
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/podmonitor.yaml
+```
+
+## WireGuard
+
+Monitor the WireGuard interfaces with:
+```shell
+kubectl create ns kilo
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/wg-exporter.yaml
+```
+
+The manifest will deploy the [Prometheus WireGuard Exporter](https://github.com/MindFlavor/prometheus_wireguard_exporter) as a DaemonSet and a [PodMonitor](https://docs.openshift.com/container-platform/4.8/rest_api/monitoring_apis/podmonitor-monitoring-coreos-com-v1.html).
+
+By default the kube-prometheus stack only monitors the `default`, `kube-system` and `monitoring` namespaces.
+In order to allow Prometheus to monitor the `kilo` namespace, apply the Role and RoleBinding with:
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/wg-exporter-role-kube-prometheus.yaml
+```
+
+## Metrics
+
+### Kilo
+
+Kilo exports some standard metrics with the Prometheus GoCollector and ProcessCollector.
+It also exposes some Kilo-specific metrics.
+
+```
+# HELP kilo_errors_total Number of errors that occurred while administering the mesh.
+# TYPE kilo_errors_total counter
+
+# HELP kilo_leader Leadership status of the node.
+# TYPE kilo_leader gauge
+
+# HELP kilo_nodes Number of nodes in the mesh.
+# TYPE kilo_nodes gauge
+
+# HELP kilo_peers Number of peers in the mesh.
+# TYPE kilo_peers gauge
+
+# HELP kilo_reconciles_total Number of reconciliation attempts.
+# TYPE kilo_reconciles_total counter
+```
+
+### WireGuard
+
+The [Prometheus WireGuard Exporter](https://github.com/MindFlavor/prometheus_wireguard_exporter) exports the following metrics:
+
+```
+# HELP wireguard_sent_bytes_total Bytes sent to the peer
+# TYPE wireguard_sent_bytes_total counter
+
+# HELP wireguard_received_bytes_total Bytes received from the peer
+# TYPE wireguard_received_bytes_total counter
+
+# HELP wireguard_latest_handshake_seconds Seconds from the last handshake
+# TYPE wireguard_latest_handshake_seconds gauge
+```
+
+## Display some Metrics
+
+If your laptop is a Kilo peer of the cluster you can access the Prometheus UI by navigating your browser directly to the cluster IP of the `prometheus-k8s` service.
+Otherwise use `port-forward`:
+```shell
+kubectl -n monitoring port-forward svc/prometheus-k8s 9090
+```
+and navigate your browser to `localhost:9090`.
+Check if you can see the PodMonitors for Kilo and the WireGuard Exporter under **Status** -> **Targets** in the Prometheus web UI.
+
+If you don't see them, check the logs of the `prometheus-k8s` Pods; it may be that Prometheus doesn't have the permission to get Pods in the `kilo` namespace.
+In this case, you need to apply the Role and RoleBinding from above.
+
+Navigate to **Graph** and try to execute a simple query, e.g. type `kilo_nodes` and click on `execute`.
+You should see some data.
+
+## Using Grafana
+
+Let's navigate to the Grafana dashboard.
+Again, if your laptop is not a Kilo peer, use `port-forward`:
+```shell
+kubectl -n monitoring port-forward svc/grafana 3000
+```
+
+Now navigate your browser to `localhost:3000`.
+The default user and password is `admin` `admin`.
+
+An example configuration for a dashboard displaying Kilo metrics can be found [here](https://raw.githubusercontent.com/squat/kilo/main/docs/grafana/kilo.json).
+You can import this dashboard by hitting **+** -> **Import** on the Grafana dashboard.
+
+The dashboard looks like this:
+
+<img src="./graphs/kilo.png" />
+

docs/network-policies.md

@@ -10,7 +10,7 @@ Support for [Kubernetes network policies](https://kubernetes.io/docs/concepts/se
 The following command adds network policy support by deploying kube-router to work alongside Kilo:
 
 ```shell
-kubectl apply -f kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kube-router.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kube-router.yaml
 ```
 
 ## Examples

docs/peer-validation.md

@@ -0,0 +1,22 @@
+# Peer Validation
+
+A [ValidatingAdmissionWebhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook) can  be used to avoid applying faulty Peer configurations to the cluster.
+
+## How It Works
+
+A [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly) is a Kubernetes resource that can be used to dynamically specify a service (i.e. the webhook server) that should validate operations (e.g. `UPDATE`, `CREATE`, etc.) on a particular resource (e.g. Kilo Peers).
+Once such a configuration is applied, the Kubernetes API server will send an AdmissionReviewRequest to the webhook service every time the specified operations are applied to the resource of the specified type.
+With regard to the [failure policy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy), the API server will apply the requested changes to a resource if the request was answered with `"allowed": true`, or deny the changes if the answer was `"allowed": false`.
+
+In case of Kilo Peer Validation, the specified operations are `UPDATE` and `CREATE`, the resources are `Peers`, and the default `failurePolicy` is set to `Fail`.
+View the full ValidatingWebhookConfiguration [here](https://github.com/squat/kilo/blob/main/manifests/peer-validation.yaml).
+
+## Getting Started
+
+Apply the Service, the Deployment of the actual webserver, and the ValidatingWebhookConfiguration with:
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/blob/main/manifests/peer-validation.yaml
+```
+
+The Kubernetes API server will only talk to webhook servers via TLS so the Kilo-Peer-Validation server must be given a valid TLS certificate and key, and the API server must be told what certificate authority (CA) to trust.
+The above manifest will use [kube-webhook-certgen](https://github.com/jet/kube-webhook-certgen) to generate the requiered certificates and patch the [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly).

docs/userspace-wireguard.md

@@ -6,29 +6,39 @@ This can make sense in cases where
 * not all nodes in a cluster have WireGuard installed; or
 * nodes are effectively immutable and kernel modules cannot be installed.
 
+One example of a userspace implementation of WireGuard is [BoringTun].
+
 ## Homogeneous Clusters
 
 In a homogeneous cluster where no node has the WireGuard kernel module, a userspace WireGuard implementation can be made available by deploying a DaemonSet.
 This DaemonSet creates a WireGuard interface that Kilo will manage.
-In order to avoid race conditions, `kg` needs to be passed the `--create-interface=false` flag. 
 
-An example configuration for a k3s cluster with [boringtun](https://github.com/cloudflare/boringtun) can be applied with:
+> **Note**: in order to avoid race conditions, `kg` needs to be passed the `--create-interface=false` flag. 
+
+An example configuration for a K3s cluster with [BoringTun] can be applied with:
 
 ```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
 kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s-userspace.yaml
 ```
 
-__Note:__ even if some nodes have the WireGuard kernel module, this configuration will cause all nodes to use the userspace implementation of WireGuard.
+> **Note**: even if some nodes have the WireGuard kernel module, this configuration will cause all nodes to use the userspace implementation of WireGuard.
 
 ## Heterogeneous Clusters
 
 In a heterogeneous cluster where some nodes are missing the WireGuard kernel module, a userspace WireGuard implementation can be provided only to the nodes that need it while enabling the other nodes to leverage WireGuard via the kernel module.
-An example of such a configuration for a k3s cluster can by applied with:
+An example of such a configuration for a K3s cluster can by applied with:
 
 ```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
 kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s-userspace-heterogeneous.yaml
 ```
 
 This configuration will deploy [nkml](https://github.com/leonnicolas/nkml) as a DaemonSet to label all nodes according to the presence of the WireGuard kernel module.
-It will also create two different DaemonSets with Kilo: `kilo` without userspace WireGuard and `kilo-userspace` with boringtun as a sidecar.
-__Note:__ because Kilo is dependant on nkml, nkml must be run on the host network before CNI is available and requires a kubeconfig in order to access the Kubernetes API.
+It will also create two different DaemonSets with Kilo:
+1. `kilo` without userspace WireGuard; and
+1. `kilo-userspace` with [BoringTun] as a sidecar.
+
+> **Note**: because Kilo is dependant on nkml, nkml must be run on the host network before CNI is available and requires a kubeconfig in order to access the Kubernetes API.
+
+[BoringTun]: https://github.com/cloudflare/boringtun

e2e/full-mesh.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_full_mesh_connectivity() {
+	assert "retry 30 5 '' check_ping" "should be able to ping all Pods"
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings"
+	echo "sleep for 30s (one reconciliation period) and try again..."
+	sleep 30
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling"
+}
+
+test_full_mesh_peer() {
+	check_peer wg99 e2e 10.5.0.1/32 full
+}
+
+test_full_mesh_allowed_location_ips() {
+	docker exec kind-cluster-kilo-control-plane ip address add 10.6.0.1/32 dev eth0
+	_kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips=10.6.0.1/32
+	assert_equals Unauthorized "$(retry 10 5 'IP is not yet routable' curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz)" "should be able to make HTTP request to allowed location IP"
+	_kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips-
+	assert "retry 10 5 'IP is still routable' _not curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz" "should not be able to make HTTP request to allowed location IP"
+	docker exec kind-cluster-kilo-control-plane ip address delete 10.6.0.1/32 dev eth0
+}
+
+test_reject_peer_empty_allowed_ips() {
+	assert_fail "create_peer e2e '' 0 foo" "should not be able to create Peer with empty allowed IPs"
+}
+
+test_reject_peer_empty_public_key() {
+	assert_fail "create_peer e2e 10.5.0.1/32 0 ''" "should not be able to create Peer with empty public key"
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity full)"
+}

e2e/handlers.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_graph_handler() {
+    assert "curl_pod 'http://10.4.0.1:1107/graph?format=svg&layout=circo' | grep -q '<svg'" "graph handler should produce SVG output"
+    assert "curl_pod http://10.4.0.1:1107/graph?layout=circo | grep -q '<svg'" "graph handler should default to SVG output"
+    assert "curl_pod http://10.4.0.1:1107/graph | grep -q '<svg'" "graph handler should default to SVG output"
+    assert_fail "curl_pod http://10.4.0.1:1107/graph?layout=fake | grep -q '<svg'" "graph handler should reject invalid layout"
+    assert_fail "curl_pod http://10.4.0.1:1107/graph?format=fake | grep -q '<svg'" "graph handler should reject invalid format"
+}
+
+test_health_handler() {
+    assert "curl_pod http://10.4.0.1:1107/health" "health handler should return a status code of 200"
+}
+
+test_metrics_handler() {
+    assert "curl_pod http://10.4.0.1:1107/metrics" "metrics handler should return a status code of 200"
+    assert "(( $(curl_pod http://10.4.0.1:1107/metrics | grep -E ^kilo_nodes | cut -d " " -f 2) > 0 ))" "metrics handler should provide metric: kilo_nodes > 0"
+}

e2e/helper-curl.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+  labels:
+    app.kubernetes.io/name: curl
+  name: curl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: curl
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: curl
+    spec:
+      containers:
+      - command:
+        - /bin/sh
+        - -c
+        - while [ 1 -eq 1 ] ; do sleep 10; done
+        image: curlimages/curl
+        name: curl
+      restartPolicy: Always

e2e/kgctl.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_connect() {
+	local PEER=test
+	local ALLOWED_IP=10.5.0.1/32
+        docker run -d --name="$PEER" --rm --network=host --cap-add=NET_ADMIN -v "$KGCTL_BINARY":/kgctl -v "$PWD/$KUBECONFIG":/kubeconfig --entrypoint=/kgctl alpine --kubeconfig /kubeconfig connect "$PEER" --allowed-ip "$ALLOWED_IP"
+	assert "retry 10 5 '' check_ping --local" "should be able to ping Pods from host"
+        docker stop "$PEER"
+}

e2e/kilo-kind-userspace.yaml

@@ -0,0 +1,203 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.3.1",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:test
+        imagePullPolicy: Never
+        args:
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --mesh-granularity=full
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+      - name: boringtun
+        image: leonnicolas/boringtun:alpine
+        args:
+        - --disable-drop-privileges=true
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo:test
+        imagePullPolicy: Never
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig

e2e/kind-config.yaml

@@ -0,0 +1,10 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane$WORKERS
+networking:
+  disableDefaultCNI: true # disable kindnet
+  apiServerAddress: 172.18.0.1
+  apiServerPort: $API_SERVER_PORT
+  podSubnet: $POD_SUBNET
+  serviceSubnet: $SERVICE_SUBNET

e2e/lib.sh

@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+KUBECONFIG="kind.yaml"
+KIND_CLUSTER="kind-cluster-kilo"
+KIND_BINARY="${KIND_BINARY:-kind}"
+KUBECTL_BINARY="${KUBECTL_BINARY:-kubectl}"
+KGCTL_BINARY="${KGCTL_BINARY:-kgctl}"
+KILO_IMAGE="${KILO_IMAGE:-squat/kilo}"
+
+retry() {
+	local COUNT="${1:-10}"
+	local SLEEP="${2:-5}"
+	local ERROR=$3
+	[ -n "$ERROR" ] && ERROR="$ERROR "
+	shift 3
+	for c in $(seq 1 "$COUNT"); do
+		if "$@"; then
+			return 0
+		else
+			printf "%s(attempt %d/%d)\n" "$ERROR" "$c" "$COUNT" | color "$YELLOW" 1>&2
+			if [ "$c" != "$COUNT" ]; then
+				printf "retrying in %d seconds...\n" "$SLEEP" | color "$YELLOW" 1>&2
+				sleep "$SLEEP"
+			fi
+		fi
+	done
+	return 1
+}
+
+_not() {
+	if "$@"; then
+		return 1
+	fi
+	return 0
+}
+
+# _kubectl is a helper that calls kubectl with the --kubeconfig flag.
+_kubectl() {
+	$KUBECTL_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# _kgctl is a helper that calls kgctl with the --kubeconfig flag.
+_kgctl() {
+	$KGCTL_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# _kind is a helper that calls kind with the --kubeconfig flag.
+_kind() {
+	$KIND_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# shellcheck disable=SC2120
+build_kind_config() {
+	local WORKER_COUNT="${1:-0}"
+	export API_SERVER_PORT="${2:-6443}"
+	export POD_SUBNET="${3:-10.42.0.0/16}"
+	export SERVICE_SUBNET="${4:-10.43.0.0/16}"
+	export WORKERS="" 
+	local i=0
+	while [ "$i" -lt "$WORKER_COUNT" ]; do
+		WORKERS="$(printf "%s\n- role: worker" "$WORKERS")"
+		((i++))
+	done
+	envsubst < ./kind-config.yaml
+	unset API_SERVER_PORT POD_SUBNET SERVICE_SUBNET WORKERS
+}
+
+create_interface() {
+	docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun --foreground --disable-drop-privileges true "$1"
+}
+
+delete_interface() {
+	docker rm --force "$1"
+}
+
+create_peer() {
+	cat <<EOF | _kubectl apply -f -
+apiVersion: kilo.squat.ai/v1alpha1
+kind: Peer
+metadata:
+  name: $1
+spec:
+  allowedIPs:
+  - $2
+  persistentKeepalive: $3
+  publicKey: $4
+EOF
+}
+
+delete_peer() {
+	_kubectl delete peer "$1"
+}
+
+is_ready() {
+	for pod in $(_kubectl -n "$1" get pods -o name -l "$2"); do
+		if ! _kubectl -n "$1" get "$pod" | tail -n 1 | grep -q Running; then
+			return 1;
+		fi
+	done
+	return 0
+}
+
+# Returns non zero if one pod of the given name in the given namespace is not ready.
+block_until_ready_by_name() {
+	block_until_ready "$1" "app.kubernetes.io/name=$2"
+}
+
+# Blocks until all pods of a deployment are ready.
+block_until_ready() {
+	retry 30 5 "some $2 pods are not ready yet" is_ready "$1" "$2"
+}
+
+
+# create_cluster launches a kind cluster and deploys Kilo, Adjacency, and a helper with curl.
+create_cluster() {
+	# shellcheck disable=SC2119
+	local CONFIG="${1:-$(build_kind_config)}"
+	_kind delete clusters $KIND_CLUSTER > /dev/null
+	# Create the kind cluster.
+	_kind create cluster --name $KIND_CLUSTER --config <(echo "$CONFIG")
+	# Load the Kilo image into kind.
+	docker tag "$KILO_IMAGE" squat/kilo:test
+	# This command does not accept the --kubeconfig flag, so call the command directly.
+	$KIND_BINARY load docker-image squat/kilo:test --name $KIND_CLUSTER
+	# Create the kubeconfig secret.
+	_kubectl create secret generic kubeconfig --from-file=kubeconfig="$KUBECONFIG" -n kube-system
+	# Apply Kilo the the cluster.
+	_kubectl apply -f ../manifests/crds.yaml
+	_kubectl apply -f kilo-kind-userspace.yaml
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait nodes --all --for=condition=Ready
+	# Wait for CoreDNS.
+	block_until_ready kube_system k8s-app=kube-dns
+	# Ensure the curl helper is not scheduled on a control-plane node.
+	_kubectl apply -f helper-curl.yaml
+	block_until_ready_by_name default curl
+	_kubectl taint node $KIND_CLUSTER-control-plane node-role.kubernetes.io/master:NoSchedule-
+	_kubectl apply -f https://raw.githubusercontent.com/kilo-io/adjacency/main/example.yaml
+	block_until_ready_by_name default adjacency
+}
+
+delete_cluster () {
+	_kind delete clusters $KIND_CLUSTER
+}
+
+curl_pod() {
+	_kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /usr/bin/curl "$@"
+}
+
+check_ping() {
+	local LOCAL
+	while [ $# -gt 0 ]; do
+		case $1 in
+			--local)
+			LOCAL=true
+			;;
+		esac
+		shift
+	done
+
+	for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		if [ -n "$LOCAL" ]; then
+			ping=$(curl -m 1 -s http://"$ip":8080/ping)
+		else
+			ping=$(curl_pod -m 1 -s http://"$ip":8080/ping)
+		fi
+		if [ "$ping" = "pong" ]; then
+			echo "successfully pinged $ip"
+		else
+			printf 'failed to ping %s; expected "pong" but got "%s"\n' "$ip" "$ping"
+			return 1
+		fi
+	done
+	return 0
+}
+
+check_adjacent() {
+	curl_pod adjacency:8080/?format=fancy
+	[ "$(curl_pod -m 1 -s adjacency:8080/?format=json | jq '.[].latencies[].ok' | grep -c true)" -eq $(($1*$1)) ]
+}
+
+check_peer() {
+	local INTERFACE=$1
+	local PEER=$2
+	local ALLOWED_IP=$3
+	local GRANULARITY=$4
+	create_interface "$INTERFACE"
+	docker run --rm leonnicolas/wg-tools wg genkey > "$INTERFACE"
+	assert "create_peer $PEER $ALLOWED_IP 10 $(docker run --rm --entrypoint=/bin/sh -v "$PWD/$INTERFACE":/key leonnicolas/wg-tools -c 'cat /key | wg pubkey')" "should be able to create Peer"
+	assert "_kgctl showconf peer $PEER --mesh-granularity=$GRANULARITY > $PEER.ini" "should be able to get Peer configuration"
+	assert "docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v $PWD/$PEER.ini:/peer.ini leonnicolas/wg-tools setconf $INTERFACE /peer.ini" "should be able to apply configuration from kgctl"
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v "$PWD/$INTERFACE":/key leonnicolas/wg-tools set "$INTERFACE" private-key /key
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip leonnicolas/wg-tools address add "$ALLOWED_IP" dev "$INTERFACE"
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip leonnicolas/wg-tools link set "$INTERFACE" up
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip leonnicolas/wg-tools route add 10.42/16 dev "$INTERFACE"
+	assert "retry 10 5 '' check_ping --local" "should be able to ping Pods from host"
+	assert_equals "$(_kgctl showconf peer "$PEER")" "$(_kgctl showconf peer "$PEER" --mesh-granularity="$GRANULARITY")" "kgctl should be able to auto detect the mesh granularity"
+	rm "$INTERFACE" "$PEER".ini
+	delete_peer "$PEER"
+	delete_interface "$INTERFACE"
+}

e2e/location-mesh.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_location_mesh_connectivity() {
+	assert "retry 30 5 '' check_ping" "should be able to ping all Pods"
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings"
+	echo "sleep for 30s (one reconciliation period) and try again..."
+	sleep 30
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling"
+}
+
+test_location_mesh_peer() {
+	check_peer wg99 e2e 10.5.0.1/32 location
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity location)"
+}

e2e/multi-cluster.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+# shellcheck disable=SC2153
+KUBECONFIG2="$KUBECONFIG"2
+# shellcheck disable=SC2153
+KIND_CLUSTER2="$KIND_CLUSTER"2
+
+setup_suite() {
+	KUBECONFIG=$KUBECONFIG2 KIND_CLUSTER=$KIND_CLUSTER2 create_cluster "$(build_kind_config 1 6444 10.44.0.0/16 10.45.0.0/16)"
+	# shellcheck disable=SC2016
+	KUBECONFIG=$KUBECONFIG2 _kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full","--subnet=10.6.0.0/16"]}]}}}}'
+	KUBECONFIG=$KUBECONFIG2 block_until_ready_by_name kube-system kilo-userspace 
+	# Register the nodes in cluster1 as peers of cluster2.
+	for n in $(_kubectl get no -o name | cut -d'/' -f2); do
+		# Specify the service CIDR as an extra IP range that should be routable.
+		$KGCTL_BINARY --kubeconfig "$KUBECONFIG" showconf node "$n" --as-peer -o yaml --allowed-ips 10.43.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG2" apply -f -
+	done
+	# Register the nodes in cluster2 as peers of cluster1.
+	for n in $(KUBECONFIG=$KUBECONFIG2 _kubectl get no -o name | cut -d'/' -f2); do
+		# Specify the service CIDR as an extra IP range that should be routable.
+		$KGCTL_BINARY --kubeconfig "$KUBECONFIG2" showconf node "$n" --as-peer -o yaml --allowed-ips 10.45.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG" apply -f -
+	done
+}
+
+test_multi_cluster_pod_connectivity() {
+	for ip in $(KUBECONFIG=$KUBECONFIG2 _kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		assert_equals pong "$(retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 1 to Pod in cluster 2"
+	done
+	for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		assert_equals pong "$(KUBECONFIG="$KUBECONFIG2" retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 2 to Pod in cluster 1"
+	done
+}
+
+test_multi_cluster_service_connectivity() {
+	# Mirror the Kubernetes API service from cluster1 into cluster2.
+	cat <<EOF | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG2" apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: mirrored-kubernetes
+spec:
+  ports:
+    - port: 443
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+    name: mirrored-kubernetes
+subsets:
+  - addresses:
+      - ip: $(_kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}') # The cluster IP of the Kubernetes API service on cluster1.
+    ports:
+      - port: 443
+EOF
+        assert_equals ok "$(KUBECONFIG="$KUBECONFIG2" retry 10 5 "service is not yet routable" curl_pod -m 1 -s -k https://mirrored-kubernetes/readyz)" "should be able to make HTTP request from cluster 2 to service in cluster 1"
+}
+
+teardown_suite () {
+	# Remove the nodes in cluster2 as peers of cluster1.
+	for n in $(KUBECONFIG=$KUBECONFIG2 _kubectl get no -o name | cut -d'/' -f2); do
+		_kubectl delete peer "$n"
+	done
+	KUBECONFIG=$KUBECONFIG2 KIND_CLUSTER=$KIND_CLUSTER2 delete_cluster
+}

e2e/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	create_cluster "$(build_kind_config 2)"
+}

e2e/teardown.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+teardown_suite () {
+	delete_cluster
+}

go.mod

@@ -1,66 +1,83 @@
 module github.com/squat/kilo
 
-go 1.15
+go 1.17
 
 require (
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/ant31/crd-validation v0.0.0-20180801212718-38f6a293f140
 	github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310
 	github.com/campoy/embedmd v1.0.0
 	github.com/containernetworking/cni v0.6.0
 	github.com/containernetworking/plugins v0.6.0
 	github.com/coreos/go-iptables v0.4.0
-	github.com/emicklei/go-restful v2.9.3+incompatible // indirect
-	github.com/evanphx/json-patch v4.2.0+incompatible // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-kit/kit v0.8.0
-	github.com/go-logfmt/logfmt v0.4.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.0 // indirect
-	github.com/go-openapi/jsonreference v0.19.0 // indirect
-	github.com/go-openapi/spec v0.19.0
-	github.com/go-openapi/swag v0.19.0 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect
-	github.com/golang/protobuf v1.3.1 // indirect
-	github.com/google/gofuzz v1.0.0 // indirect
-	github.com/googleapis/gnostic v0.2.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.0 // indirect
+	github.com/go-kit/kit v0.9.0
 	github.com/imdario/mergo v0.3.6 // indirect
-	github.com/json-iterator/go v1.1.6 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
-	github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/oklog/run v1.0.0
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/prometheus/client_golang v0.9.2
-	github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267
-	github.com/spf13/pflag v1.0.3
-	github.com/stretchr/testify v1.3.0 // indirect
+	github.com/oklog/run v1.1.0
+	github.com/prometheus/client_golang v1.11.0
+	github.com/spf13/cobra v1.1.3
 	github.com/vishvananda/netlink v1.0.0
 	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
-	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
-	golang.org/x/lint v0.0.0-20190409202823-959b441ac422
-	golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 // indirect
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
-	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872
-	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
-	golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 // indirect
-	google.golang.org/appengine v1.5.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	k8s.io/api v0.0.0-20190313235455-40a48860b5ab
-	k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed
-	k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1
-	k8s.io/client-go v11.0.0+incompatible
-	k8s.io/code-generator v0.0.0-20190311093542-50b561225d70
-	k8s.io/gengo v0.0.0-20181106084056-51747d6e00da // indirect
-	k8s.io/klog v0.3.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30
-	k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 // indirect
-	sigs.k8s.io/yaml v1.1.0 // indirect
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
+	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211124212657-dd7407c86d22
+	k8s.io/api v0.21.1
+	k8s.io/apiextensions-apiserver v0.21.1
+	k8s.io/apimachinery v0.21.1
+	k8s.io/client-go v0.21.1
+	k8s.io/code-generator v0.21.1
+	sigs.k8s.io/controller-tools v0.6.0
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/evanphx/json-patch v4.9.0+incompatible // indirect
+	github.com/fatih/color v1.12.0 // indirect
+	github.com/go-logfmt/logfmt v0.5.0 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/gobuffalo/flect v0.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.4.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mdlayher/genetlink v1.0.0 // indirect
+	github.com/mdlayher/netlink v1.4.1 // indirect
+	github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.26.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	golang.org/x/tools v0.1.2 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20211123210315-387f7c461a16 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 // indirect
+	k8s.io/klog/v2 v2.8.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
 )

go.sum

@@ -1,207 +1,824 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
+github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/ant31/crd-validation v0.0.0-20180801212718-38f6a293f140 h1:ljF9TLBK+JYXKB8aklLhcubPDBUjpvzWKiqgZ37s+QY=
-github.com/ant31/crd-validation v0.0.0-20180801212718-38f6a293f140/go.mod h1:X0noFIik9YqfhGYBLEHg8LJKEwy7QIitLQuFMpKLcPk=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U=
 github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
 github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containernetworking/cni v0.6.0 h1:FXICGBZNMtdHlW65trpoHviHctQD3seWhRRcqp2hMOU=
 github.com/containernetworking/cni v0.6.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/plugins v0.6.0 h1:bqPT7yYisnWs+FrtgY5/qLEB9QZ/6z11wMNCwSdzZm0=
 github.com/containernetworking/plugins v0.6.0/go.mod h1:dagHaAhNjXjT9QYOklkKJDGaQPTg4pf//FrUcJeb7FU=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.4.0 h1:wh4UbVs8DhLUbpyq97GLJDKrQMjEDD63T1xE4CrsKzQ=
 github.com/coreos/go-iptables v0.4.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emicklei/go-restful v2.9.3+incompatible h1:2OwhVdhtzYUp5P5wuGsVDPagKSRd9JK72sJCHVCXh5g=
-github.com/emicklei/go-restful v2.9.3+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
-github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=
+github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
+github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
-github.com/go-openapi/jsonpointer v0.19.0 h1:FTUMcX77w5rQkClIzDtTxvn6Bsa894CcrzNj2MMfeg8=
-github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
-github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk=
-github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/spec v0.19.0 h1:A4SZ6IWh3lnjH0rG0Z5lkxazMGBECtrZcbyYQi+64k4=
-github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/swag v0.19.0 h1:Kg7Wl7LkTPlmc393QZQ/5rQadPhi7pBVEMZxyTi0Ii8=
-github.com/go-openapi/swag v0.19.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gobuffalo/flect v0.2.2 h1:PAVD7sp0KOdfswjAw9BpLCU9hXo7wFSzgpQ+zNeks/A=
+github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg=
-github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
-github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA=
+github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
+github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
+github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
+github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw=
+github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs=
+github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA=
+github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U=
+github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190 h1:iycCSDo8EKVueI9sfVBBJmtNn9DnXV/K1YWwEJO+uOs=
+github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43 h1:WgyLFv10Ov49JAQI/ZLUkCZ7VJS3r74hwFIGXJsgZlY=
+github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
+github.com/mdlayher/genetlink v1.0.0 h1:OoHN1OdyEIkScEmRgxLEe2M9U8ClMytqA5niynLtfj0=
+github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc=
+github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
+github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
+github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
+github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o=
+github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8=
+github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
+github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
+github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys=
+github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
+github.com/mdlayher/netlink v1.4.1 h1:I154BCU+mKlIf7BgcAJB2r7QjveNPty6uNY1g9ChVfI=
+github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
+github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
+github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb h1:2dC7L10LmTqlyMVzFJ00qM25lqESg9Z4u3GuEXN5iHY=
+github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
+github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
-github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
+github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
+github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
+github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
-github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267 h1:I9j1PLS64+NgCtkgbomGInboj1NFH1KF1tkVKlt3yF4=
-github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
 github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI=
+golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYRuq8JQ1aa7LJt8EXVyo=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI=
+golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
+golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d/go.mod h1:5yyfuiqVIJ7t+3MqrpTQ+QqRkMWiESiyDvPNvKYCecg=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
+golang.zx2c4.com/wireguard v0.0.0-20211123210315-387f7c461a16 h1:SCBV/ayxt56AuC0R8oN5p8Mmc9Llv34tr9VbNlh1kL0=
+golang.zx2c4.com/wireguard v0.0.0-20211123210315-387f7c461a16/go.mod h1:TjUWrnD5ATh7bFvmm/ALEJZQ4ivKbETb6pmyj1vUoNI=
+golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211124212657-dd7407c86d22 h1:TnoJ6AWs/q4xGE9smgTi+bJmEDet3nrBqdHSV0d5/zA=
+golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211124212657-dd7407c86d22/go.mod h1:ELrVb7MHNIb8OV6iZ6TP/ScunqUha+vWsUP+SVBH7lQ=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-k8s.io/api v0.0.0-20190313235455-40a48860b5ab h1:DG9A67baNpoeweOy2spF1OWHhnVY5KR7/Ek/+U1lVZc=
-k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
-k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed h1:rCteec//ELIjZMfjIGQbVtZooyaofqDJwsmWwWKItNs=
-k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
-k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 h1:IS7K02iBkQXpCeieSiyJjGoLSdVOv2DbPaWHJ+ZtgKg=
-k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
-k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
-k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
-k8s.io/code-generator v0.0.0-20190311093542-50b561225d70 h1:lgPp615xLHxN84RBd+viA/oHzJfI0miFYFH4T9wpPQ4=
-k8s.io/code-generator v0.0.0-20190311093542-50b561225d70/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8=
-k8s.io/gengo v0.0.0-20181106084056-51747d6e00da h1:ZMvcXtMVbhUCtCuiSEzBV+Eur4swzfdxx6ZyX3qT6dk=
-k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE=
-k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
-k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 h1:8r+l4bNWjRlsFYlQJnKJ2p7s1YQPj4XyXiJVqDHRx7c=
-k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
-sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c=
+k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
+k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo=
+k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA=
+k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs=
+k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
+k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY=
+k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4=
+k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
+k8s.io/code-generator v0.21.1 h1:jvcxHpVu5dm/LMXr3GOj/jroiP8+v2YnJE9i2OVRenk=
+k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
+k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
+k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/controller-tools v0.6.0 h1:o2Fm1K7CmIp8OVaBtXsWB/ssBAzyoKZPPAGR3VuxaKs=
+sigs.k8s.io/controller-tools v0.6.0/go.mod h1:baRMVPrctU77F+rfAuH2uPqW93k6yQnZA2dhUOr7ihc=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

manifests/crds.yaml

@@ -0,0 +1,92 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0
+  creationTimestamp: null
+  name: peers.kilo.squat.ai
+spec:
+  group: kilo.squat.ai
+  names:
+    kind: Peer
+    listKind: PeerList
+    plural: peers
+    singular: peer
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Peer is a WireGuard peer that should have access to the VPN.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: 'Specification of the desired behavior of the Kilo Peer.
+              More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status'
+            properties:
+              allowedIPs:
+                description: AllowedIPs is the list of IP addresses that are allowed
+                  for the given peer's tunnel.
+                items:
+                  type: string
+                type: array
+              endpoint:
+                description: Endpoint is the initial endpoint for connections to the
+                  peer.
+                properties:
+                  dnsOrIP:
+                    description: DNSOrIP is a DNS name or an IP address.
+                    properties:
+                      dns:
+                        description: DNS must be a valid RFC 1123 subdomain.
+                        type: string
+                      ip:
+                        description: IP must be a valid IP address.
+                        type: string
+                    type: object
+                  port:
+                    description: Port must be a valid port number.
+                    format: int32
+                    type: integer
+                required:
+                - dnsOrIP
+                - port
+                type: object
+              persistentKeepalive:
+                description: PersistentKeepalive is the interval in seconds of the
+                  emission of keepalive packets by the peer. This defaults to 0, which
+                  disables the feature.
+                type: integer
+              presharedKey:
+                description: PresharedKey is the optional symmetric encryption key
+                  for the peer.
+                type: string
+              publicKey:
+                description: PublicKey is the WireGuard public key for the peer.
+                type: string
+            required:
+            - allowedIPs
+            - publicKey
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

manifests/kilo-bootkube-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,20 +51,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,6 +79,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-bootkube.yaml

@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,20 +85,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +110,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -126,7 +131,7 @@ spec:
           readOnly: false
       initContainers:
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         command:
         - /bin/sh
         - -c

manifests/kilo-k3s-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -45,6 +44,35 @@ subjects:
     name: kilo
     namespace: kube-system
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -52,20 +80,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,13 +108,16 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
         - name: kilo-dir
           mountPath: /var/lib/kilo
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
         - name: lib-modules
           mountPath: /lib/modules
@@ -91,6 +125,28 @@ spec:
         - name: xtables-lock
           mountPath: /run/xtables.lock
           readOnly: false
+      initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.4.1
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       tolerations:
       - effect: NoSchedule
         operator: Exists
@@ -101,11 +157,13 @@ spec:
         hostPath:
           path: /var/lib/kilo
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # Since kilo runs as a daemonset, it is recommended that you copy the
-          # k3s.yaml kubeconfig file from the master node to all worker nodes
-          # with the same path structure.
-          path: /etc/rancher/k3s/k3s.yaml
+          path: /var/lib/rancher/k3s/agent
       - name: lib-modules
         hostPath:
           path: /lib/modules

manifests/kilo-k3s-userspace-heterogeneous.yaml

@@ -58,14 +58,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -80,6 +79,35 @@ subjects:
     name: kilo
     namespace: kube-system
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -87,14 +115,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       nodeSelector:
         nkml.squat.ai/wireguard: "true"
@@ -102,7 +133,7 @@ spec:
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -112,6 +143,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -120,7 +154,7 @@ spec:
         - name: kilo-dir
           mountPath: /var/lib/kilo
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
         - name: lib-modules
           mountPath: /lib/modules
@@ -129,8 +163,29 @@ spec:
           mountPath: /run/xtables.lock
           readOnly: false
       initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.4.1
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         command:
         - /bin/sh
         - -c
@@ -169,11 +224,13 @@ spec:
         hostPath:
           path: /var/lib/kilo
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # Since kilo runs as a daemonset, it is recommended that you copy the
-          # k3s.yaml kubeconfig file from the master node to all worker nodes
-          # with the same path structure.
-          path: /etc/rancher/k3s/k3s.yaml
+          path: /var/lib/rancher/k3s/agent
       - name: lib-modules
         hostPath:
           path: /lib/modules
@@ -189,14 +246,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
     spec:
       nodeSelector:
         nkml.squat.ai/wireguard: "false"
@@ -204,7 +264,7 @@ spec:
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo 
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -215,6 +275,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -223,7 +286,7 @@ spec:
         - name: kilo-dir
           mountPath: /var/lib/kilo
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
         - name: lib-modules
           mountPath: /lib/modules
@@ -247,8 +310,29 @@ spec:
           mountPath: /var/run/wireguard
           readOnly: false
       initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.4.1
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         command:
         - /bin/sh
         - -c
@@ -287,11 +371,13 @@ spec:
         hostPath:
           path: /var/lib/kilo
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # Since kilo runs as a daemonset, it is recommended that you copy the
-          # k3s.yaml kubeconfig file from the master node to all worker nodes
-          # with the same path structure.
-          path: /etc/rancher/k3s/k3s.yaml
+          path: /var/lib/rancher/k3s/agent
       - name: lib-modules
         hostPath:
           path: /lib/modules
@@ -320,6 +406,7 @@ spec:
         app.kubernetes.io/name: nkml
     spec:
       hostNetwork: true
+      serviceAccountName: kilo
       containers:
       - name: nkml
         image: leonnicolas/nkml
@@ -337,13 +424,36 @@ spec:
           containerPort: 8080
         volumeMounts:
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
+      initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.4.1
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       volumes:
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # since the above DaemonSets are dependant on the labels
-          # and nkml would need a cni to start
-          # it needs run on the hostnetwork and use the kubeconfig
-          # to label the nodes
-          path: /etc/rancher/k3s/k3s.yaml
+          path: /var/lib/rancher/k3s/agent

manifests/kilo-k3s-userspace.yaml

@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -79,27 +78,60 @@ subjects:
     name: kilo
     namespace: kube-system
 ---
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kilo
   namespace: kube-system
   labels:
-    app.kubernetes.io/name: kilo
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
-      app.kubernetes.io/name: kilo
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: kilo
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -110,6 +142,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -118,7 +153,7 @@ spec:
         - name: kilo-dir
           mountPath: /var/lib/kilo
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
         - name: lib-modules
           mountPath: /lib/modules
@@ -142,8 +177,29 @@ spec:
           mountPath: /var/run/wireguard
           readOnly: false
       initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.4.1
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         command:
         - /bin/sh
         - -c
@@ -182,11 +238,13 @@ spec:
         hostPath:
           path: /var/lib/kilo
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # Since kilo runs as a daemonset, it is recommended that you copy the
-          # k3s.yaml kubeconfig file from the master node to all worker nodes
-          # with the same path structure.
-          path: /etc/rancher/k3s/k3s.yaml
+          path: /var/lib/rancher/k3s/agent
       - name: lib-modules
         hostPath:
           path: /lib/modules

manifests/kilo-k3s.yaml

@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -79,6 +78,35 @@ subjects:
     name: kilo
     namespace: kube-system
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -86,20 +114,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +139,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -116,7 +150,7 @@ spec:
         - name: kilo-dir
           mountPath: /var/lib/kilo
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
         - name: lib-modules
           mountPath: /lib/modules
@@ -125,8 +159,29 @@ spec:
           mountPath: /run/xtables.lock
           readOnly: false
       initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.4.1
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         command:
         - /bin/sh
         - -c
@@ -165,11 +220,13 @@ spec:
         hostPath:
           path: /var/lib/kilo
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # Since kilo runs as a daemonset, it is recommended that you copy the
-          # k3s.yaml kubeconfig file from the master node to all worker nodes
-          # with the same path structure.
-          path: /etc/rancher/k3s/k3s.yaml
+          path: /var/lib/rancher/k3s/agent
       - name: lib-modules
         hostPath:
           path: /lib/modules

manifests/kilo-kubeadm-flannel-userspace.yaml

@@ -0,0 +1,142 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: boringtun
+        image: leonnicolas/boringtun
+        args:
+          - --disable-drop-privileges=true
+          - --foreground
+          - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: wireguard
+            mountPath: /var/run/wireguard
+            readOnly: false
+      - name: kilo
+        image: squat/kilo:0.4.1
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        - --cni=false
+        - --compatibility=flannel
+        - --local=false
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      tolerations:
+      - operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        configMap:
+          name: kube-proxy
+          items:
+          - key: kubeconfig.conf
+            path: kubeconfig
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard

manifests/kilo-kubeadm-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,20 +51,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,6 +79,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-kubeadm-userspace.yaml

@@ -0,0 +1,207 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.3.1",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: boringtun
+        image: leonnicolas/boringtun
+        imagePullPolicy: IfNotPresent
+        args:
+          - --disable-drop-privileges=true
+          - --foreground
+          - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: wireguard
+            mountPath: /var/run/wireguard
+            readOnly: false
+      - name: kilo
+        image: squat/kilo:0.4.1
+        imagePullPolicy: IfNotPresent
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo:0.4.1
+        imagePullPolicy: IfNotPresent
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        configMap:
+          name: kube-proxy
+          items:
+          - key: kubeconfig.conf
+            path: kubeconfig
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard

manifests/kilo-kubeadm.yaml

@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,20 +85,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +110,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -126,7 +131,7 @@ spec:
           readOnly: false
       initContainers:
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         command:
         - /bin/sh
         - -c

manifests/kilo-typhoon-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,20 +51,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,6 +79,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-typhoon.yaml

@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,20 +85,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +110,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -126,7 +131,7 @@ spec:
           readOnly: false
       initContainers:
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.4.1
         command:
         - /bin/sh
         - -c

manifests/peer-validation.yaml

@@ -0,0 +1,173 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kilo
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: "peers.kilo.squat.ai"
+webhooks:
+- name: "peers.kilo.squat.ai"
+  rules:
+  - apiGroups:   ["kilo.squat.ai"]
+    apiVersions: ["v1alpha1"]
+    operations:  ["CREATE","UPDATE"]
+    resources:   ["peers"]
+    scope:       "Cluster"
+  clientConfig:
+    service:
+      namespace: "kilo"
+      name: "peer-validation"
+      path: "/validate"
+  admissionReviewVersions: ["v1"]
+  sideEffects: None
+  timeoutSeconds: 5
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: peer-validation-server
+  namespace: kilo
+  labels:
+    app.kubernetes.io/name: peer-validation-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: peer-validation-server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: peer-validation-server
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+      containers:
+      - name: server
+        image: squat/kilo:0.4.1
+        args:
+        - webhook
+        - --cert-file=/run/secrets/tls/tls.crt
+        - --key-file=/run/secrets/tls/tls.key
+        - --listen-metrics=:1107
+        - --listen=:8443
+        ports:
+        - containerPort: 8443
+          name: webhook
+        - containerPort: 1107
+          name: metrics
+        volumeMounts:
+        - name: tls
+          mountPath: /run/secrets/tls
+          readOnly: true
+      volumes:
+      - name: tls
+        secret:
+          secretName: peer-validation-webhook-tls
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: peer-validation
+  namespace: kilo
+spec:
+  selector:
+    app.kubernetes.io/name: peer-validation-server
+  ports:
+    - port: 443
+      targetPort: webhook
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo-peer-validation
+  namespace: kilo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo-peer-validation
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  resourceNames:
+  - peers.kilo.squat.ai
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo-peer-validation
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo-peer-validation
+subjects:
+  - kind: ServiceAccount
+    namespace: kilo
+    name: kilo-peer-validation
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kilo-peer-validation
+  namespace: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kilo-peer-validation
+  namespace: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kilo-peer-validation
+subjects:
+  - kind: ServiceAccount
+    namespace: kilo
+    name: kilo-peer-validation
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cert-gen
+  namespace: kilo
+spec:
+  template:
+    spec:
+      serviceAccountName: kilo-peer-validation
+      initContainers:
+      - name: create
+        image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0
+        args:
+        - create
+        - --namespace=kilo
+        - --secret-name=peer-validation-webhook-tls
+        - --host=peer-validation,peer-validation.kilo.svc
+        - --key-name=tls.key
+        - --cert-name=tls.crt
+      containers:
+      - name: patch
+        image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0
+        args:
+        - patch
+        - --webhook-name=peers.kilo.squat.ai
+        - --secret-name=peer-validation-webhook-tls
+        - --namespace=kilo
+        - --patch-mutating=false
+      restartPolicy: OnFailure
+  backoffLimit: 4

manifests/podmonitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+  name: kilo
+  namespace: kilo
+spec:
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  podMetricsEndpoints:
+  - interval: 15s
+    port: metrics
+    path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: kilo

manifests/wg-exporter-role-kube-prometheus.yaml

@@ -0,0 +1,56 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: prometheus
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 2.26.0
+  name: prometheus-k8s
+  namespace: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: prometheus
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 2.26.0
+  name: prometheus-k8s
+  namespace: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: monitoring

manifests/wg-exporter.yaml

@@ -0,0 +1,67 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: wg-exporter
+    app.kubernetes.io/part-of: kilo
+  name: wg-exporter
+  namespace: kilo
+spec:
+  namespaceSelector:
+    matchNames:
+    - kilo
+  podMetricsEndpoints:
+  - interval: 15s
+    port: metrics
+    path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: kilo
+      app.kubernetes.io/name: wg-exporter
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/name: wg-exporter
+    app.kubernetes.io/part-of: kilo
+  name: wg-exporter
+  namespace: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: wg-exporter
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: wg-exporter
+        app.kubernetes.io/part-of: kilo
+    spec:
+      containers:
+      - args:
+        - -a
+        - -i=kilo0
+        - -p=9586
+        image: mindflavor/prometheus-wireguard-exporter
+        name: wg-exporter
+        ports:
+        - containerPort: 9586
+          name: metrics
+          protocol: TCP
+        securityContext:
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+      volumes:
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists

pkg/encapsulation/ipip.go

@@ -74,7 +74,7 @@ func (i *ipip) Rules(nodes []*net.IPNet) []iptables.Rule {
 	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-j", "KILO-IPIP"))
 	for _, n := range nodes {
 		// Accept encapsulated traffic from peers.
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(n.IP)), "filter", "KILO-IPIP", "-s", n.String(), "-m", "comment", "--comment", "Kilo: allow IPIP traffic", "-j", "ACCEPT"))
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(n.IP), "filter", "KILO-IPIP", "-s", n.String(), "-m", "comment", "--comment", "Kilo: allow IPIP traffic", "-j", "ACCEPT"))
 	}
 	// Drop all other IPIP traffic.
 	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-j", "DROP"))

pkg/encapsulation/ipip_cgo.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build cgo
 // +build cgo
 
 package encapsulation

pkg/encapsulation/ipip_nocgo.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cgo
 // +build !cgo
 
 package encapsulation

pkg/iptables/iptables.go

@@ -16,7 +16,9 @@ package iptables
 
 import (
 	"fmt"
+	"io"
 	"net"
+	"os"
 	"sync"
 	"time"
 
@@ -25,6 +27,21 @@ import (
 	"github.com/go-kit/kit/log/level"
 )
 
+const ipv6ModuleDisabledPath = "/sys/module/ipv6/parameters/disable"
+
+func ipv6Disabled() (bool, error) {
+	f, err := os.Open(ipv6ModuleDisabledPath)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+	disabled := make([]byte, 1)
+	if _, err = io.ReadFull(f, disabled); err != nil {
+		return false, err
+	}
+	return disabled[0] == '1', nil
+}
+
 // Protocol represents an IP protocol.
 type Protocol byte
 
@@ -36,11 +53,11 @@ const (
 )
 
 // GetProtocol will return a protocol from the length of an IP address.
-func GetProtocol(length int) Protocol {
-	if length == net.IPv6len {
-		return ProtocolIPv6
+func GetProtocol(ip net.IP) Protocol {
+	if len(ip) == net.IPv4len || ip.To4() != nil {
+		return ProtocolIPv4
 	}
-	return ProtocolIPv4
+	return ProtocolIPv6
 }
 
 // Client represents any type that can administer iptables rules.
@@ -253,11 +270,20 @@ func New(opts ...ControllerOption) (*Controller, error) {
 		c.v4 = v4
 	}
 	if c.v6 == nil {
-		v6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+		disabled, err := ipv6Disabled()
 		if err != nil {
-			return nil, fmt.Errorf("failed to create iptables IPv6 client: %v", err)
+			return nil, fmt.Errorf("failed to check IPv6 status: %v", err)
+		}
+		if disabled {
+			level.Info(c.logger).Log("msg", "IPv6 is disabled in the kernel; disabling the IPv6 iptables controller")
+			c.v6 = &fakeClient{}
+		} else {
+			v6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create iptables IPv6 client: %v", err)
+			}
+			c.v6 = v6
 		}
-		c.v6 = v6
 	}
 	return c, nil
 }

pkg/k8s/apis/kilo/register.go

@@ -16,5 +16,5 @@ package kilo
 
 const (
 	// GroupName contains the API group name for Kilo API group.
-	GroupName = "kilo"
+	GroupName = "kilo.squat.ai"
 )

pkg/k8s/apis/kilo/v1alpha1/doc.go

@@ -15,5 +15,5 @@
 // +k8s:deepcopy-gen=package,register
 
 // Package v1alpha1 is the v1alpha1 version of the API.
-// +groupName=kilo
+// +groupName=kilo.squat.ai
 package v1alpha1

pkg/k8s/apis/kilo/v1alpha1/types.go

@@ -15,6 +15,7 @@
 package v1alpha1
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -47,6 +48,7 @@ var PeerShortNames = []string{"peer"}
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +k8s:openapi-gen=true
+// +kubebuilder:resource:scope=Cluster
 
 // Peer is a WireGuard peer that should have access to the VPN.
 type Peer struct {
@@ -76,20 +78,21 @@ type PeerSpec struct {
 	PersistentKeepalive int `json:"persistentKeepalive,omitempty"`
 	// PresharedKey is the optional symmetric encryption key for the peer.
 	// +optional
-	PresharedKey string `json:"presharedKey"`
+	PresharedKey string `json:"presharedKey,omitempty"`
 	// PublicKey is the WireGuard public key for the peer.
 	PublicKey string `json:"publicKey"`
 }
 
-// PeerEndpoint represents a WireGuard enpoint, which is a ip:port tuple.
+// PeerEndpoint represents a WireGuard endpoint, which is an IP:port tuple.
 type PeerEndpoint struct {
-	DNSOrIP
+	// DNSOrIP is a DNS name or an IP address.
+	DNSOrIP `json:"dnsOrIP"`
 	// Port must be a valid port number.
 	Port uint32 `json:"port"`
 }
 
 // DNSOrIP represents either a DNS name or an IP address.
-// IPs, as they are more specific, are preferred.
+// When both are given, the IP address, as it is more specific, override the DNS name.
 type DNSOrIP struct {
 	// DNS must be a valid RFC 1123 subdomain.
 	// +optional
@@ -133,7 +136,7 @@ func (p *Peer) Copy() *Peer {
 func (p *Peer) Validate() error {
 	for _, ip := range p.Spec.AllowedIPs {
 		if _, n, err := net.ParseCIDR(ip); err != nil {
-			return fmt.Errorf("failed to parse %q as a valid IP address: %v", ip, err)
+			return fmt.Errorf("failed to parse %q as a valid IP address: %w", ip, err)
 		} else if n == nil {
 			return fmt.Errorf("got invalid IP address for %q", ip)
 		}
@@ -157,8 +160,11 @@ func (p *Peer) Validate() error {
 	if p.Spec.PersistentKeepalive < 0 {
 		return fmt.Errorf("persistent keepalive must be greater than or equal to zero; got %q", p.Spec.PersistentKeepalive)
 	}
-	if len(p.Spec.PublicKey) == 0 {
-		return errors.New("public keys cannot be empty")
+	if b, err := base64.StdEncoding.DecodeString(p.Spec.PublicKey); err != nil {
+		return fmt.Errorf("WireGuard public key is not base64 encoded: %w", err)
+		// Since WireGuard is using Curve25519 for the key exchange, the key length of 256 bits should not change in the near future.
+	} else if len(b) != 32 {
+		return errors.New("WireGuard public key has invalid length")
 	}
 	return nil
 }
@@ -168,6 +174,9 @@ func (p *Peer) Validate() error {
 // PeerList is a list of peers.
 type PeerList struct {
 	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	// +optional
 	metav1.ListMeta `json:"metadata,omitempty"`
 	// List of peers.
 	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md

pkg/k8s/apis/kilo/v1alpha1/zz_generated.deepcopy.go

@@ -1,6 +1,7 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -86,7 +87,7 @@ func (in *PeerEndpoint) DeepCopy() *PeerEndpoint {
 func (in *PeerList) DeepCopyInto(out *PeerList) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
-	out.ListMeta = in.ListMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
 	if in.Items != nil {
 		in, out := &in.Items, &out.Items
 		*out = make([]Peer, len(*in))

pkg/k8s/backend.go

@@ -15,6 +15,7 @@
 package k8s
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -24,15 +25,15 @@ import (
 	"strings"
 	"time"
 
-	crdutils "github.com/ant31/crd-validation/pkg"
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
-	"k8s.io/apimachinery/pkg/util/validation"
 	v1informers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
 	v1listers "k8s.io/client-go/listers/core/v1"
@@ -59,12 +60,17 @@ const (
 	locationAnnotationKey        = "kilo.squat.ai/location"
 	persistentKeepaliveKey       = "kilo.squat.ai/persistent-keepalive"
 	wireGuardIPAnnotationKey     = "kilo.squat.ai/wireguard-ip"
+	discoveredEndpointsKey       = "kilo.squat.ai/discovered-endpoints"
+	allowedLocationIPsKey        = "kilo.squat.ai/allowed-location-ips"
+	granularityKey               = "kilo.squat.ai/granularity"
 	// RegionLabelKey is the key for the well-known Kubernetes topology region label.
 	RegionLabelKey  = "topology.kubernetes.io/region"
 	jsonPatchSlash  = "~1"
 	jsonRemovePatch = `{"op": "remove", "path": "%s"}`
 )
 
+var logger = log.NewNopLogger()
+
 type backend struct {
 	nodes *nodeBackend
 	peers *peerBackend
@@ -97,10 +103,12 @@ type peerBackend struct {
 }
 
 // New creates a new instance of a mesh.Backend.
-func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Interface, topologyLabel string) mesh.Backend {
+func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Interface, topologyLabel string, l log.Logger) mesh.Backend {
 	ni := v1informers.NewNodeInformer(c, 5*time.Minute, nil)
 	pi := v1alpha1informers.NewPeerInformer(kc, 5*time.Minute, nil)
 
+	logger = l
+
 	return &backend{
 		&nodeBackend{
 			client:        c,
@@ -127,8 +135,10 @@ func (nb *nodeBackend) CleanUp(name string) error {
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(keyAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(lastSeenAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(wireGuardIPAnnotationKey, "/", jsonPatchSlash, 1))),
+		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(discoveredEndpointsKey, "/", jsonPatchSlash, 1))),
+		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(granularityKey, "/", jsonPatchSlash, 1))),
 	}, ",") + "]")
-	if _, err := nb.client.CoreV1().Nodes().Patch(name, types.JSONPatchType, patch); err != nil {
+	if _, err := nb.client.CoreV1().Nodes().Patch(context.TODO(), name, types.JSONPatchType, patch, metav1.PatchOptions{}); err != nil {
 		return fmt.Errorf("failed to patch node: %v", err)
 	}
 	return nil
@@ -214,13 +224,23 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
 	} else {
 		n.ObjectMeta.Annotations[internalIPAnnotationKey] = node.InternalIP.String()
 	}
-	n.ObjectMeta.Annotations[keyAnnotationKey] = string(node.Key)
+	n.ObjectMeta.Annotations[keyAnnotationKey] = node.Key.String()
 	n.ObjectMeta.Annotations[lastSeenAnnotationKey] = strconv.FormatInt(node.LastSeen, 10)
 	if node.WireGuardIP == nil {
 		n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = ""
 	} else {
 		n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = node.WireGuardIP.String()
 	}
+	if node.DiscoveredEndpoints == nil {
+		n.ObjectMeta.Annotations[discoveredEndpointsKey] = ""
+	} else {
+		discoveredEndpoints, err := json.Marshal(node.DiscoveredEndpoints)
+		if err != nil {
+			return err
+		}
+		n.ObjectMeta.Annotations[discoveredEndpointsKey] = string(discoveredEndpoints)
+	}
+	n.ObjectMeta.Annotations[granularityKey] = string(node.Granularity)
 	oldData, err := json.Marshal(old)
 	if err != nil {
 		return err
@@ -233,7 +253,7 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
 	if err != nil {
 		return fmt.Errorf("failed to create patch for node %q: %v", n.Name, err)
 	}
-	if _, err = nb.client.CoreV1().Nodes().Patch(name, types.StrategicMergePatchType, patch); err != nil {
+	if _, err = nb.client.CoreV1().Nodes().Patch(context.TODO(), name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
 		return fmt.Errorf("failed to patch node: %v", err)
 	}
 	return nil
@@ -262,9 +282,9 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 		location = node.ObjectMeta.Labels[topologyLabel]
 	}
 	// Allow the endpoint to be overridden.
-	endpoint := parseEndpoint(node.ObjectMeta.Annotations[forceEndpointAnnotationKey])
+	endpoint := wireguard.ParseEndpoint(node.ObjectMeta.Annotations[forceEndpointAnnotationKey])
 	if endpoint == nil {
-		endpoint = parseEndpoint(node.ObjectMeta.Annotations[endpointAnnotationKey])
+		endpoint = wireguard.ParseEndpoint(node.ObjectMeta.Annotations[endpointAnnotationKey])
 	}
 	// Allow the internal IP to be overridden.
 	internalIP := normalizeIP(node.ObjectMeta.Annotations[forceInternalIPAnnotationKey])
@@ -278,13 +298,11 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 		internalIP = nil
 	}
 	// Set Wireguard PersistentKeepalive setting for the node.
-	var persistentKeepalive int64
-	if keepAlive, ok := node.ObjectMeta.Annotations[persistentKeepaliveKey]; !ok {
-		persistentKeepalive = 0
-	} else {
-		if persistentKeepalive, err = strconv.ParseInt(keepAlive, 10, 64); err != nil {
-			persistentKeepalive = 0
-		}
+	var persistentKeepalive time.Duration
+	if keepAlive, ok := node.ObjectMeta.Annotations[persistentKeepaliveKey]; ok {
+		// We can ignore the error, because p will be set to 0 if an error occures.
+		p, _ := strconv.ParseInt(keepAlive, 10, 64)
+		persistentKeepalive = time.Duration(p) * time.Second
 	}
 	var lastSeen int64
 	if ls, ok := node.ObjectMeta.Annotations[lastSeenAnnotationKey]; !ok {
@@ -294,6 +312,36 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 			lastSeen = 0
 		}
 	}
+	var discoveredEndpoints map[string]*net.UDPAddr
+	if de, ok := node.ObjectMeta.Annotations[discoveredEndpointsKey]; ok {
+		err := json.Unmarshal([]byte(de), &discoveredEndpoints)
+		if err != nil {
+			discoveredEndpoints = nil
+		}
+	}
+	// Set allowed IPs for a location.
+	var allowedLocationIPs []net.IPNet
+	if str, ok := node.ObjectMeta.Annotations[allowedLocationIPsKey]; ok {
+		for _, ip := range strings.Split(str, ",") {
+			if ipnet := normalizeIP(ip); ipnet != nil {
+				allowedLocationIPs = append(allowedLocationIPs, *ipnet)
+			}
+		}
+	}
+	var meshGranularity mesh.Granularity
+	if gr, ok := node.ObjectMeta.Annotations[granularityKey]; ok {
+		meshGranularity = mesh.Granularity(gr)
+		switch meshGranularity {
+		case mesh.LogicalGranularity:
+		case mesh.FullGranularity:
+		default:
+			meshGranularity = ""
+		}
+	}
+
+	// TODO log some error or warning.
+	key, _ := wgtypes.ParseKey(node.ObjectMeta.Annotations[keyAnnotationKey])
+
 	return &mesh.Node{
 		// Endpoint and InternalIP should only ever fail to parse if the
 		// remote node's agent has not yet set its IP address;
@@ -304,17 +352,20 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 		Endpoint:            endpoint,
 		NoInternalIP:        noInternalIP,
 		InternalIP:          internalIP,
-		Key:                 []byte(node.ObjectMeta.Annotations[keyAnnotationKey]),
+		Key:                 key,
 		LastSeen:            lastSeen,
 		Leader:              leader,
 		Location:            location,
 		Name:                node.Name,
-		PersistentKeepalive: int(persistentKeepalive),
+		PersistentKeepalive: persistentKeepalive,
 		Subnet:              subnet,
 		// WireGuardIP can fail to parse if the node is not a leader or if
 		// the node's agent has not yet reconciled. In either case, the IP
 		// will parse as nil.
-		WireGuardIP: normalizeIP(node.ObjectMeta.Annotations[wireGuardIPAnnotationKey]),
+		WireGuardIP:         normalizeIP(node.ObjectMeta.Annotations[wireGuardIPAnnotationKey]),
+		DiscoveredEndpoints: discoveredEndpoints,
+		AllowedLocationIPs:  allowedLocationIPs,
+		Granularity:         meshGranularity,
 	}
 }
 
@@ -323,14 +374,14 @@ func translatePeer(peer *v1alpha1.Peer) *mesh.Peer {
 	if peer == nil {
 		return nil
 	}
-	var aips []*net.IPNet
+	var aips []net.IPNet
 	for _, aip := range peer.Spec.AllowedIPs {
 		aip := normalizeIP(aip)
 		// Skip any invalid IPs.
 		if aip == nil {
 			continue
 		}
-		aips = append(aips, aip)
+		aips = append(aips, *aip)
 	}
 	var endpoint *wireguard.Endpoint
 	if peer.Spec.Endpoint != nil {
@@ -340,36 +391,41 @@ func translatePeer(peer *v1alpha1.Peer) *mesh.Peer {
 		} else {
 			ip = ip.To16()
 		}
-		if peer.Spec.Endpoint.Port > 0 && (ip != nil || peer.Spec.Endpoint.DNS != "") {
-			endpoint = &wireguard.Endpoint{
-				DNSOrIP: wireguard.DNSOrIP{
-					DNS: peer.Spec.Endpoint.DNS,
-					IP:  ip,
-				},
-				Port: peer.Spec.Endpoint.Port,
+		if peer.Spec.Endpoint.Port > 0 {
+			if ip != nil {
+				endpoint = wireguard.NewEndpoint(ip, int(peer.Spec.Endpoint.Port))
+			}
+			if peer.Spec.Endpoint.DNS != "" {
+				endpoint = wireguard.ParseEndpoint(fmt.Sprintf("%s:%d", peer.Spec.Endpoint.DNS, peer.Spec.Endpoint.Port))
 			}
 		}
 	}
-	var key []byte
-	if len(peer.Spec.PublicKey) > 0 {
-		key = []byte(peer.Spec.PublicKey)
+
+	key, err := wgtypes.ParseKey(peer.Spec.PublicKey)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to parse public key", "peer", peer.Name, "err", err.Error())
 	}
-	var psk []byte
-	if len(peer.Spec.PresharedKey) > 0 {
-		psk = []byte(peer.Spec.PresharedKey)
+	var psk *wgtypes.Key
+	if k, err := wgtypes.ParseKey(peer.Spec.PresharedKey); err != nil {
+		// Set key to nil to avoid setting a key to the zero value wgtypes.Key{}
+		psk = nil
+	} else {
+		psk = &k
 	}
-	var pka int
+	var pka time.Duration
 	if peer.Spec.PersistentKeepalive > 0 {
-		pka = peer.Spec.PersistentKeepalive
+		pka = time.Duration(peer.Spec.PersistentKeepalive) * time.Second
 	}
 	return &mesh.Peer{
 		Name: peer.Name,
 		Peer: wireguard.Peer{
-			AllowedIPs:          aips,
-			Endpoint:            endpoint,
-			PersistentKeepalive: pka,
-			PresharedKey:        psk,
-			PublicKey:           key,
+			PeerConfig: wgtypes.PeerConfig{
+				AllowedIPs:                  aips,
+				PersistentKeepaliveInterval: &pka,
+				PresharedKey:                psk,
+				PublicKey:                   key,
+			},
+			Endpoint: endpoint,
 		},
 	}
 }
@@ -391,24 +447,9 @@ func (pb *peerBackend) Get(name string) (*mesh.Peer, error) {
 // Init initializes the backend; for this backend that means
 // syncing the informer cache.
 func (pb *peerBackend) Init(stop <-chan struct{}) error {
-	// Register CRD.
-	crd := crdutils.NewCustomResourceDefinition(crdutils.Config{
-		SpecDefinitionName:    "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1.Peer",
-		EnableValidation:      true,
-		ResourceScope:         string(v1beta1.ClusterScoped),
-		Group:                 v1alpha1.GroupName,
-		Kind:                  v1alpha1.PeerKind,
-		Version:               v1alpha1.SchemeGroupVersion.Version,
-		Plural:                v1alpha1.PeerPlural,
-		ShortNames:            v1alpha1.PeerShortNames,
-		GetOpenAPIDefinitions: v1alpha1.GetOpenAPIDefinitions,
-	})
-	crd.Spec.Subresources.Scale = nil
-	crd.Spec.Subresources.Status = nil
-
-	_, err := pb.extensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
-	if err != nil && !apierrors.IsAlreadyExists(err) {
-		return fmt.Errorf("failed to create CRD: %v", err)
+	// Check the presents of the CRD peers.kilo.squat.ai.
+	if _, err := pb.extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), strings.Join([]string{v1alpha1.PeerPlural, v1alpha1.GroupName}, "."), metav1.GetOptions{}); err != nil {
+		return fmt.Errorf("CRD is not present: %v", err)
 	}
 
 	go pb.informer.Run(stop)
@@ -482,22 +523,26 @@ func (pb *peerBackend) Set(name string, peer *mesh.Peer) error {
 		p.Spec.AllowedIPs[i] = peer.AllowedIPs[i].String()
 	}
 	if peer.Endpoint != nil {
-		var ip string
-		if peer.Endpoint.IP != nil {
-			ip = peer.Endpoint.IP.String()
-		}
 		p.Spec.Endpoint = &v1alpha1.PeerEndpoint{
 			DNSOrIP: v1alpha1.DNSOrIP{
-				IP:  ip,
-				DNS: peer.Endpoint.DNS,
+				IP:  peer.Endpoint.IP().String(),
+				DNS: peer.Endpoint.DNS(),
 			},
-			Port: peer.Endpoint.Port,
+			Port: uint32(peer.Endpoint.Port()),
 		}
 	}
-	p.Spec.PersistentKeepalive = peer.PersistentKeepalive
-	p.Spec.PresharedKey = string(peer.PresharedKey)
-	p.Spec.PublicKey = string(peer.PublicKey)
-	if _, err = pb.client.KiloV1alpha1().Peers().Update(p); err != nil {
+	if peer.PersistentKeepaliveInterval == nil {
+		p.Spec.PersistentKeepalive = 0
+	} else {
+		p.Spec.PersistentKeepalive = int(*peer.PersistentKeepaliveInterval / time.Second)
+	}
+	if peer.PresharedKey == nil {
+		p.Spec.PresharedKey = ""
+	} else {
+		p.Spec.PresharedKey = peer.PresharedKey.String()
+	}
+	p.Spec.PublicKey = peer.PublicKey.String()
+	if _, err = pb.client.KiloV1alpha1().Peers().Update(context.TODO(), p, metav1.UpdateOptions{}); err != nil {
 		return fmt.Errorf("failed to update peer: %v", err)
 	}
 	return nil
@@ -520,35 +565,3 @@ func normalizeIP(ip string) *net.IPNet {
 	ipNet.IP = i.To16()
 	return ipNet
 }
-
-func parseEndpoint(endpoint string) *wireguard.Endpoint {
-	if len(endpoint) == 0 {
-		return nil
-	}
-	parts := strings.Split(endpoint, ":")
-	if len(parts) < 2 {
-		return nil
-	}
-	portRaw := parts[len(parts)-1]
-	hostRaw := strings.Trim(strings.Join(parts[:len(parts)-1], ":"), "[]")
-	port, err := strconv.ParseUint(portRaw, 10, 32)
-	if err != nil {
-		return nil
-	}
-	if len(validation.IsValidPortNum(int(port))) != 0 {
-		return nil
-	}
-	ip := net.ParseIP(hostRaw)
-	if ip == nil {
-		if len(validation.IsDNS1123Subdomain(hostRaw)) == 0 {
-			return &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{DNS: hostRaw}, Port: uint32(port)}
-		}
-		return nil
-	}
-	if ip4 := ip.To4(); ip4 != nil {
-		ip = ip4
-	} else {
-		ip = ip.To16()
-	}
-	return &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: ip}, Port: uint32(port)}
-}

pkg/k8s/backend_test.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,8 +17,10 @@ package k8s
 import (
 	"net"
 	"testing"
+	"time"
 
 	"github.com/kylelemons/godebug/pretty"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
@@ -26,6 +28,30 @@ import (
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
+func mustKey() (k wgtypes.Key) {
+	var err error
+	if k, err = wgtypes.GeneratePrivateKey(); err != nil {
+		panic(err.Error())
+	}
+	return
+}
+
+func mustPSKKey() (key *wgtypes.Key) {
+	if k, err := wgtypes.GenerateKey(); err != nil {
+		panic(err.Error())
+	} else {
+		key = &k
+	}
+	return
+}
+
+var (
+	fooKey = mustKey()
+	pskKey = mustPSKKey()
+	second = time.Second
+	zero   = time.Duration(0)
+)
+
 func TestTranslateNode(t *testing.T) {
 	for _, tc := range []struct {
 		name        string
@@ -54,8 +80,19 @@ func TestTranslateNode(t *testing.T) {
 				internalIPAnnotationKey: "10.0.0.2/32",
 			},
 			out: &mesh.Node{
-				Endpoint:   &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: mesh.DefaultKiloPort},
-				InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
+				Endpoint:   wireguard.NewEndpoint(net.ParseIP("10.0.0.1").To4(), mesh.DefaultKiloPort),
+				InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2").To4(), Mask: net.CIDRMask(32, 32)},
+			},
+		},
+		{
+			name: "valid ips with ipv6",
+			annotations: map[string]string{
+				endpointAnnotationKey:   "[ff10::10]:51820",
+				internalIPAnnotationKey: "ff60::10/64",
+			},
+			out: &mesh.Node{
+				Endpoint:   wireguard.NewEndpoint(net.ParseIP("ff10::10").To16(), mesh.DefaultKiloPort),
+				InternalIP: &net.IPNet{IP: net.ParseIP("ff60::10").To16(), Mask: net.CIDRMask(64, 128)},
 			},
 		},
 		{
@@ -68,7 +105,7 @@ func TestTranslateNode(t *testing.T) {
 			name:        "normalize subnet",
 			annotations: map[string]string{},
 			out: &mesh.Node{
-				Subnet: &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(24, 32)},
+				Subnet: &net.IPNet{IP: net.ParseIP("10.2.0.0").To4(), Mask: net.CIDRMask(24, 32)},
 			},
 			subnet: "10.2.0.1/24",
 		},
@@ -76,7 +113,7 @@ func TestTranslateNode(t *testing.T) {
 			name:        "valid subnet",
 			annotations: map[string]string{},
 			out: &mesh.Node{
-				Subnet: &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				Subnet: &net.IPNet{IP: net.ParseIP("10.2.1.0").To4(), Mask: net.CIDRMask(24, 32)},
 			},
 			subnet: "10.2.1.0/24",
 		},
@@ -108,7 +145,7 @@ func TestTranslateNode(t *testing.T) {
 				forceEndpointAnnotationKey: "-10.0.0.2:51821",
 			},
 			out: &mesh.Node{
-				Endpoint: &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: mesh.DefaultKiloPort},
+				Endpoint: wireguard.NewEndpoint(net.ParseIP("10.0.0.1").To4(), mesh.DefaultKiloPort),
 			},
 		},
 		{
@@ -118,7 +155,7 @@ func TestTranslateNode(t *testing.T) {
 				forceEndpointAnnotationKey: "10.0.0.2:51821",
 			},
 			out: &mesh.Node{
-				Endpoint: &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.2")}, Port: 51821},
+				Endpoint: wireguard.NewEndpoint(net.ParseIP("10.0.0.2").To4(), 51821),
 			},
 		},
 		{
@@ -127,7 +164,7 @@ func TestTranslateNode(t *testing.T) {
 				persistentKeepaliveKey: "25",
 			},
 			out: &mesh.Node{
-				PersistentKeepalive: 25,
+				PersistentKeepalive: 25 * time.Second,
 			},
 		},
 		{
@@ -137,7 +174,7 @@ func TestTranslateNode(t *testing.T) {
 				forceInternalIPAnnotationKey: "-10.1.0.2/24",
 			},
 			out: &mesh.Node{
-				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.1"), Mask: net.CIDRMask(24, 32)},
+				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.1").To4(), Mask: net.CIDRMask(24, 32)},
 				NoInternalIP: false,
 			},
 		},
@@ -148,7 +185,7 @@ func TestTranslateNode(t *testing.T) {
 				forceInternalIPAnnotationKey: "10.1.0.2/24",
 			},
 			out: &mesh.Node{
-				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(24, 32)},
+				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.2").To4(), Mask: net.CIDRMask(24, 32)},
 				NoInternalIP: false,
 			},
 		},
@@ -166,7 +203,7 @@ func TestTranslateNode(t *testing.T) {
 				forceEndpointAnnotationKey:   "10.0.0.2:51821",
 				forceInternalIPAnnotationKey: "10.1.0.2/32",
 				internalIPAnnotationKey:      "10.1.0.1/32",
-				keyAnnotationKey:             "foo",
+				keyAnnotationKey:             fooKey.String(),
 				lastSeenAnnotationKey:        "1000000000",
 				leaderAnnotationKey:          "",
 				locationAnnotationKey:        "b",
@@ -177,14 +214,45 @@ func TestTranslateNode(t *testing.T) {
 				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
-				Endpoint:            &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.2")}, Port: 51821},
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("10.0.0.2").To4(), 51821),
 				NoInternalIP:        false,
-				InternalIP:          &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(32, 32)},
-				Key:                 []byte("foo"),
+				InternalIP:          &net.IPNet{IP: net.ParseIP("10.1.0.2").To4(), Mask: net.CIDRMask(32, 32)},
+				Key:                 fooKey,
 				LastSeen:            1000000000,
 				Leader:              true,
 				Location:            "b",
-				PersistentKeepalive: 25,
+				PersistentKeepalive: 25 * time.Second,
+				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0").To4(), Mask: net.CIDRMask(24, 32)},
+				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1").To4(), Mask: net.CIDRMask(16, 32)},
+			},
+			subnet: "10.2.1.0/24",
+		},
+		{
+			name: "complete with ipv6",
+			annotations: map[string]string{
+				endpointAnnotationKey:        "10.0.0.1:51820",
+				forceEndpointAnnotationKey:   "[1100::10]:51821",
+				forceInternalIPAnnotationKey: "10.1.0.2/32",
+				internalIPAnnotationKey:      "10.1.0.1/32",
+				keyAnnotationKey:             fooKey.String(),
+				lastSeenAnnotationKey:        "1000000000",
+				leaderAnnotationKey:          "",
+				locationAnnotationKey:        "b",
+				persistentKeepaliveKey:       "25",
+				wireGuardIPAnnotationKey:     "10.4.0.1/16",
+			},
+			labels: map[string]string{
+				RegionLabelKey: "a",
+			},
+			out: &mesh.Node{
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("1100::10"), 51821),
+				NoInternalIP:        false,
+				InternalIP:          &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(32, 32)},
+				Key:                 fooKey,
+				LastSeen:            1000000000,
+				Leader:              true,
+				Location:            "b",
+				PersistentKeepalive: 25 * time.Second,
 				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
 				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
 			},
@@ -195,7 +263,7 @@ func TestTranslateNode(t *testing.T) {
 			annotations: map[string]string{
 				endpointAnnotationKey:    "10.0.0.1:51820",
 				internalIPAnnotationKey:  "",
-				keyAnnotationKey:         "foo",
+				keyAnnotationKey:         fooKey.String(),
 				lastSeenAnnotationKey:    "1000000000",
 				locationAnnotationKey:    "b",
 				persistentKeepaliveKey:   "25",
@@ -205,13 +273,13 @@ func TestTranslateNode(t *testing.T) {
 				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
-				Endpoint:            &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: 51820},
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("10.0.0.1"), 51820),
 				InternalIP:          nil,
-				Key:                 []byte("foo"),
+				Key:                 fooKey,
 				LastSeen:            1000000000,
 				Leader:              false,
 				Location:            "b",
-				PersistentKeepalive: 25,
+				PersistentKeepalive: 25 * time.Second,
 				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
 				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
 			},
@@ -223,7 +291,7 @@ func TestTranslateNode(t *testing.T) {
 				endpointAnnotationKey:        "10.0.0.1:51820",
 				internalIPAnnotationKey:      "10.1.0.1/32",
 				forceInternalIPAnnotationKey: "",
-				keyAnnotationKey:             "foo",
+				keyAnnotationKey:             fooKey.String(),
 				lastSeenAnnotationKey:        "1000000000",
 				locationAnnotationKey:        "b",
 				persistentKeepaliveKey:       "25",
@@ -233,14 +301,14 @@ func TestTranslateNode(t *testing.T) {
 				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
-				Endpoint:            &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: 51820},
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("10.0.0.1"), 51820),
 				NoInternalIP:        true,
 				InternalIP:          nil,
-				Key:                 []byte("foo"),
+				Key:                 fooKey,
 				LastSeen:            1000000000,
 				Leader:              false,
 				Location:            "b",
-				PersistentKeepalive: 25,
+				PersistentKeepalive: 25 * time.Second,
 				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
 				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
 			},
@@ -266,7 +334,13 @@ func TestTranslatePeer(t *testing.T) {
 	}{
 		{
 			name: "empty",
-			out:  &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "invalid ips",
@@ -276,7 +350,13 @@ func TestTranslatePeer(t *testing.T) {
 					"foo",
 				},
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "valid ips",
@@ -288,9 +368,12 @@ func TestTranslatePeer(t *testing.T) {
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					AllowedIPs: []*net.IPNet{
-						{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(24, 32)},
-						{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
+					PeerConfig: wgtypes.PeerConfig{
+						AllowedIPs: []net.IPNet{
+							{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(24, 32)},
+							{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
+						},
+						PersistentKeepaliveInterval: &zero,
 					},
 				},
 			},
@@ -305,7 +388,13 @@ func TestTranslatePeer(t *testing.T) {
 					Port: mesh.DefaultKiloPort,
 				},
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "only endpoint port",
@@ -314,7 +403,13 @@ func TestTranslatePeer(t *testing.T) {
 					Port: mesh.DefaultKiloPort,
 				},
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "valid endpoint ip",
@@ -328,10 +423,29 @@ func TestTranslatePeer(t *testing.T) {
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					Endpoint: &wireguard.Endpoint{
-						DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")},
-						Port:    mesh.DefaultKiloPort,
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
 					},
+					Endpoint: wireguard.NewEndpoint(net.ParseIP("10.0.0.1").To4(), mesh.DefaultKiloPort),
+				},
+			},
+		},
+		{
+			name: "valid endpoint ipv6",
+			spec: v1alpha1.PeerSpec{
+				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
+						IP: "ff60::2",
+					},
+					Port: mesh.DefaultKiloPort,
+				},
+			},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+					Endpoint: wireguard.NewEndpoint(net.ParseIP("ff60::2").To16(), mesh.DefaultKiloPort),
 				},
 			},
 		},
@@ -347,9 +461,9 @@ func TestTranslatePeer(t *testing.T) {
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					Endpoint: &wireguard.Endpoint{
-						DNSOrIP: wireguard.DNSOrIP{DNS: "example.com"},
-						Port:    mesh.DefaultKiloPort,
+					Endpoint: wireguard.ParseEndpoint("example.com:51820"),
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
 					},
 				},
 			},
@@ -359,16 +473,25 @@ func TestTranslatePeer(t *testing.T) {
 			spec: v1alpha1.PeerSpec{
 				PublicKey: "",
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "valid key",
 			spec: v1alpha1.PeerSpec{
-				PublicKey: "foo",
+				PublicKey: fooKey.String(),
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					PublicKey: []byte("foo"),
+					PeerConfig: wgtypes.PeerConfig{
+						PublicKey:                   fooKey,
+						PersistentKeepaliveInterval: &zero,
+					},
 				},
 			},
 		},
@@ -377,7 +500,13 @@ func TestTranslatePeer(t *testing.T) {
 			spec: v1alpha1.PeerSpec{
 				PersistentKeepalive: -1,
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "valid keepalive",
@@ -386,18 +515,23 @@ func TestTranslatePeer(t *testing.T) {
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					PersistentKeepalive: 1,
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &second,
+					},
 				},
 			},
 		},
 		{
 			name: "valid preshared key",
 			spec: v1alpha1.PeerSpec{
-				PresharedKey: "psk",
+				PresharedKey: pskKey.String(),
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					PresharedKey: []byte("psk"),
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+						PresharedKey:                pskKey,
+					},
 				},
 			},
 		},
@@ -410,52 +544,3 @@ func TestTranslatePeer(t *testing.T) {
 		}
 	}
 }
-
-func TestParseEndpoint(t *testing.T) {
-	for _, tc := range []struct {
-		name     string
-		endpoint string
-		out      *wireguard.Endpoint
-	}{
-		{
-			name:     "empty",
-			endpoint: "",
-			out:      nil,
-		},
-		{
-			name:     "invalid IP",
-			endpoint: "10.0.0.:51820",
-			out:      nil,
-		},
-		{
-			name:     "invalid hostname",
-			endpoint: "foo-:51820",
-			out:      nil,
-		},
-		{
-			name:     "invalid port",
-			endpoint: "10.0.0.1:100000000",
-			out:      nil,
-		},
-		{
-			name:     "valid IP",
-			endpoint: "10.0.0.1:51820",
-			out:      &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: mesh.DefaultKiloPort},
-		},
-		{
-			name:     "valid IPv6",
-			endpoint: "[ff02::114]:51820",
-			out:      &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("ff02::114")}, Port: mesh.DefaultKiloPort},
-		},
-		{
-			name:     "valid hostname",
-			endpoint: "foo:51821",
-			out:      &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{DNS: "foo"}, Port: 51821},
-		},
-	} {
-		endpoint := parseEndpoint(tc.endpoint)
-		if diff := pretty.Compare(endpoint, tc.out); diff != "" {
-			t.Errorf("test case %q: got diff: %v", tc.name, diff)
-		}
-	}
-}

pkg/k8s/clientset/versioned/clientset.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 package versioned
 
 import (
+	"fmt"
+
 	kilov1alpha1 "github.com/squat/kilo/pkg/k8s/clientset/versioned/typed/kilo/v1alpha1"
 	discovery "k8s.io/client-go/discovery"
 	rest "k8s.io/client-go/rest"
@@ -49,9 +51,14 @@ func (c *Clientset) Discovery() discovery.DiscoveryInterface {
 }
 
 // NewForConfig creates a new Clientset for the given config.
+// If config's RateLimiter is not set and QPS and Burst are acceptable,
+// NewForConfig will generate a rate-limiter in configShallowCopy.
 func NewForConfig(c *rest.Config) (*Clientset, error) {
 	configShallowCopy := *c
 	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
+		if configShallowCopy.Burst <= 0 {
+			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
+		}
 		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
 	}
 	var cs Clientset

pkg/k8s/clientset/versioned/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/fake/clientset_generated.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -39,7 +39,7 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset {
 		}
 	}
 
-	cs := &Clientset{}
+	cs := &Clientset{tracker: o}
 	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
 	cs.AddReactor("*", "*", testing.ObjectReaction(o))
 	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
@@ -61,12 +61,17 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset {
 type Clientset struct {
 	testing.Fake
 	discovery *fakediscovery.FakeDiscovery
+	tracker   testing.ObjectTracker
 }
 
 func (c *Clientset) Discovery() discovery.DiscoveryInterface {
 	return c.discovery
 }
 
+func (c *Clientset) Tracker() testing.ObjectTracker {
+	return c.tracker
+}
+
 var _ clientset.Interface = &Clientset{}
 
 // KiloV1alpha1 retrieves the KiloV1alpha1Client

pkg/k8s/clientset/versioned/fake/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/fake/register.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ import (
 
 var scheme = runtime.NewScheme()
 var codecs = serializer.NewCodecFactory(scheme)
-var parameterCodec = runtime.NewParameterCodec(scheme)
+
 var localSchemeBuilder = runtime.SchemeBuilder{
 	kilov1alpha1.AddToScheme,
 }

pkg/k8s/clientset/versioned/scheme/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/scheme/register.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/fake_kilo_client.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/fake_peer.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 package fake
 
 import (
+	"context"
+
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	labels "k8s.io/apimachinery/pkg/labels"
@@ -31,12 +33,12 @@ type FakePeers struct {
 	Fake *FakeKiloV1alpha1
 }
 
-var peersResource = schema.GroupVersionResource{Group: "kilo", Version: "v1alpha1", Resource: "peers"}
+var peersResource = schema.GroupVersionResource{Group: "kilo.squat.ai", Version: "v1alpha1", Resource: "peers"}
 
-var peersKind = schema.GroupVersionKind{Group: "kilo", Version: "v1alpha1", Kind: "Peer"}
+var peersKind = schema.GroupVersionKind{Group: "kilo.squat.ai", Version: "v1alpha1", Kind: "Peer"}
 
 // Get takes name of the peer, and returns the corresponding peer object, and an error if there is any.
-func (c *FakePeers) Get(name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootGetAction(peersResource, name), &v1alpha1.Peer{})
 	if obj == nil {
@@ -46,7 +48,7 @@ func (c *FakePeers) Get(name string, options v1.GetOptions) (result *v1alpha1.Pe
 }
 
 // List takes label and field selectors, and returns the list of Peers that match those selectors.
-func (c *FakePeers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
+func (c *FakePeers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootListAction(peersResource, peersKind, opts), &v1alpha1.PeerList{})
 	if obj == nil {
@@ -67,13 +69,13 @@ func (c *FakePeers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err er
 }
 
 // Watch returns a watch.Interface that watches the requested peers.
-func (c *FakePeers) Watch(opts v1.ListOptions) (watch.Interface, error) {
+func (c *FakePeers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
 	return c.Fake.
 		InvokesWatch(testing.NewRootWatchAction(peersResource, opts))
 }
 
 // Create takes the representation of a peer and creates it.  Returns the server's representation of the peer, and an error, if there is any.
-func (c *FakePeers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootCreateAction(peersResource, peer), &v1alpha1.Peer{})
 	if obj == nil {
@@ -83,7 +85,7 @@ func (c *FakePeers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err erro
 }
 
 // Update takes the representation of a peer and updates it. Returns the server's representation of the peer, and an error, if there is any.
-func (c *FakePeers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootUpdateAction(peersResource, peer), &v1alpha1.Peer{})
 	if obj == nil {
@@ -93,22 +95,22 @@ func (c *FakePeers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err erro
 }
 
 // Delete takes name of the peer and deletes it. Returns an error if one occurs.
-func (c *FakePeers) Delete(name string, options *v1.DeleteOptions) error {
+func (c *FakePeers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	_, err := c.Fake.
 		Invokes(testing.NewRootDeleteAction(peersResource, name), &v1alpha1.Peer{})
 	return err
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *FakePeers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
-	action := testing.NewRootDeleteCollectionAction(peersResource, listOptions)
+func (c *FakePeers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	action := testing.NewRootDeleteCollectionAction(peersResource, listOpts)
 
 	_, err := c.Fake.Invokes(action, &v1alpha1.PeerList{})
 	return err
 }
 
 // Patch applies the patch and returns the patched peer.
-func (c *FakePeers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootPatchSubresourceAction(peersResource, name, pt, data, subresources...), &v1alpha1.Peer{})
 	if obj == nil {

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/generated_expansion.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/kilo_client.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ package v1alpha1
 import (
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
 	"github.com/squat/kilo/pkg/k8s/clientset/versioned/scheme"
-	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
 	rest "k8s.io/client-go/rest"
 )
 
@@ -28,7 +27,7 @@ type KiloV1alpha1Interface interface {
 	PeersGetter
 }
 
-// KiloV1alpha1Client is used to interact with features provided by the kilo group.
+// KiloV1alpha1Client is used to interact with features provided by the kilo.squat.ai group.
 type KiloV1alpha1Client struct {
 	restClient rest.Interface
 }
@@ -69,7 +68,7 @@ func setConfigDefaults(config *rest.Config) error {
 	gv := v1alpha1.SchemeGroupVersion
 	config.GroupVersion = &gv
 	config.APIPath = "/apis"
-	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
+	config.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
 
 	if config.UserAgent == "" {
 		config.UserAgent = rest.DefaultKubernetesUserAgent()

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package v1alpha1
 
 import (
+	"context"
 	"time"
 
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
@@ -35,14 +36,14 @@ type PeersGetter interface {
 
 // PeerInterface has methods to work with Peer resources.
 type PeerInterface interface {
-	Create(*v1alpha1.Peer) (*v1alpha1.Peer, error)
-	Update(*v1alpha1.Peer) (*v1alpha1.Peer, error)
-	Delete(name string, options *v1.DeleteOptions) error
-	DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
-	Get(name string, options v1.GetOptions) (*v1alpha1.Peer, error)
-	List(opts v1.ListOptions) (*v1alpha1.PeerList, error)
-	Watch(opts v1.ListOptions) (watch.Interface, error)
-	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error)
+	Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (*v1alpha1.Peer, error)
+	Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (*v1alpha1.Peer, error)
+	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
+	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
+	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.Peer, error)
+	List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.PeerList, error)
+	Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error)
 	PeerExpansion
 }
 
@@ -59,19 +60,19 @@ func newPeers(c *KiloV1alpha1Client) *peers {
 }
 
 // Get takes name of the peer, and returns the corresponding peer object, and an error if there is any.
-func (c *peers) Get(name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
+func (c *peers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Get().
 		Resource("peers").
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // List takes label and field selectors, and returns the list of Peers that match those selectors.
-func (c *peers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
+func (c *peers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -81,13 +82,13 @@ func (c *peers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error)
 		Resource("peers").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Watch returns a watch.Interface that watches the requested peers.
-func (c *peers) Watch(opts v1.ListOptions) (watch.Interface, error) {
+func (c *peers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -97,66 +98,69 @@ func (c *peers) Watch(opts v1.ListOptions) (watch.Interface, error) {
 		Resource("peers").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Watch()
+		Watch(ctx)
 }
 
 // Create takes the representation of a peer and creates it.  Returns the server's representation of the peer, and an error, if there is any.
-func (c *peers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *peers) Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Post().
 		Resource("peers").
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(peer).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Update takes the representation of a peer and updates it. Returns the server's representation of the peer, and an error, if there is any.
-func (c *peers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *peers) Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Put().
 		Resource("peers").
 		Name(peer.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(peer).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Delete takes name of the peer and deletes it. Returns an error if one occurs.
-func (c *peers) Delete(name string, options *v1.DeleteOptions) error {
+func (c *peers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	return c.client.Delete().
 		Resource("peers").
 		Name(name).
-		Body(options).
-		Do().
+		Body(&opts).
+		Do(ctx).
 		Error()
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *peers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+func (c *peers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
 	var timeout time.Duration
-	if listOptions.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second
+	if listOpts.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
 	}
 	return c.client.Delete().
 		Resource("peers").
-		VersionedParams(&listOptions, scheme.ParameterCodec).
+		VersionedParams(&listOpts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Body(options).
-		Do().
+		Body(&opts).
+		Do(ctx).
 		Error()
 }
 
 // Patch applies the patch and returns the patched peer.
-func (c *peers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error) {
+func (c *peers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Patch(pt).
 		Resource("peers").
-		SubResource(subresources...).
 		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(data).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }

pkg/k8s/informers/factory.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/generic.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ func (f *genericInformer) Lister() cache.GenericLister {
 // TODO extend this to unknown resources with a client pool
 func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {
 	switch resource {
-	// Group=kilo, Version=v1alpha1
+	// Group=kilo.squat.ai, Version=v1alpha1
 	case v1alpha1.SchemeGroupVersion.WithResource("peers"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Kilo().V1alpha1().Peers().Informer()}, nil
 

pkg/k8s/informers/internalinterfaces/factory_interfaces.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/kilo/interface.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/kilo/v1alpha1/interface.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/kilo/v1alpha1/peer.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package v1alpha1
 
 import (
+	"context"
 	time "time"
 
 	kilov1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
@@ -58,13 +59,13 @@ func NewFilteredPeerInformer(client versioned.Interface, resyncPeriod time.Durat
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.KiloV1alpha1().Peers().List(options)
+				return client.KiloV1alpha1().Peers().List(context.TODO(), options)
 			},
 			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.KiloV1alpha1().Peers().Watch(options)
+				return client.KiloV1alpha1().Peers().Watch(context.TODO(), options)
 			},
 		},
 		&kilov1alpha1.Peer{},

pkg/k8s/listers/kilo/v1alpha1/expansion_generated.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/listers/kilo/v1alpha1/peer.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -24,10 +24,13 @@ import (
 )
 
 // PeerLister helps list Peers.
+// All objects returned here must be treated as read-only.
 type PeerLister interface {
 	// List lists all Peers in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1alpha1.Peer, err error)
 	// Get retrieves the Peer from the index for a given name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1alpha1.Peer, error)
 	PeerListerExpansion
 }

pkg/mesh/backend.go

@@ -18,6 +18,8 @@ import (
 	"net"
 	"time"
 
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
@@ -47,12 +49,15 @@ const (
 	// FullGranularity indicates that the network should create
 	// a mesh between every node.
 	FullGranularity Granularity = "full"
+	// AutoGranularity can be used with kgctl to obtain
+	// the granularity automatically.
+	AutoGranularity Granularity = "auto"
 )
 
 // Node represents a node in the network.
 type Node struct {
 	Endpoint     *wireguard.Endpoint
-	Key          []byte
+	Key          wgtypes.Key
 	NoInternalIP bool
 	InternalIP   *net.IPNet
 	// LastSeen is a Unix time for the last time
@@ -63,15 +68,23 @@ type Node struct {
 	Leader              bool
 	Location            string
 	Name                string
-	PersistentKeepalive int
+	PersistentKeepalive time.Duration
 	Subnet              *net.IPNet
 	WireGuardIP         *net.IPNet
+	// DiscoveredEndpoints cannot be DNS endpoints, only net.UDPAddr.
+	DiscoveredEndpoints map[string]*net.UDPAddr
+	AllowedLocationIPs  []net.IPNet
+	Granularity         Granularity
 }
 
 // Ready indicates whether or not the node is ready.
 func (n *Node) Ready() bool {
 	// Nodes that are not leaders will not have WireGuardIPs, so it is not required.
-	return n != nil && n.Endpoint != nil && !(n.Endpoint.IP == nil && n.Endpoint.DNS == "") && n.Endpoint.Port != 0 && n.Key != nil && n.Subnet != nil && time.Now().Unix()-n.LastSeen < int64(checkInPeriod)*2/int64(time.Second)
+	return n != nil &&
+		n.Endpoint.Ready() &&
+		n.Key != wgtypes.Key{} &&
+		n.Subnet != nil &&
+		time.Now().Unix()-n.LastSeen < int64(checkInPeriod)*2/int64(time.Second)
 }
 
 // Peer represents a peer in the network.
@@ -86,7 +99,10 @@ type Peer struct {
 // will not declare their endpoint and instead allow it to be
 // discovered.
 func (p *Peer) Ready() bool {
-	return p != nil && p.AllowedIPs != nil && len(p.AllowedIPs) != 0 && p.PublicKey != nil
+	return p != nil &&
+		p.AllowedIPs != nil &&
+		len(p.AllowedIPs) != 0 &&
+		p.PublicKey != wgtypes.Key{} // If Key was not set, it will be wgtypes.Key{}.
 }
 
 // EventType describes what kind of an action an event represents.

pkg/mesh/cni.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build linux
 // +build linux
 
 package mesh

pkg/mesh/discoverips.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build linux
 // +build linux
 
 package mesh
@@ -59,6 +60,7 @@ func getIP(hostname string, ignoreIfaces ...int) (*net.IPNet, *net.IPNet, error)
 			ignore[oneAddressCIDR(ip.IP).String()] = struct{}{}
 		}
 	}
+
 	var hostPriv, hostPub []*net.IPNet
 	{
 		// Check IPs to which hostname resolves first.
@@ -71,6 +73,9 @@ func getIP(hostname string, ignoreIfaces ...int) (*net.IPNet, *net.IPNet, error)
 			if !ok {
 				continue
 			}
+			if isLocal(ip.IP) {
+				continue
+			}
 			ip.Mask = mask
 			if isPublic(ip.IP) {
 				hostPub = append(hostPub, ip)

pkg/mesh/graph.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"github.com/awalterschulze/gographviz"
+
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
@@ -166,8 +167,9 @@ func nodeLabel(location, name string, cidr *net.IPNet, priv, wgIP net.IP, endpoi
 	if wgIP != nil {
 		label = append(label, wgIP.String())
 	}
-	if endpoint != nil {
-		label = append(label, endpoint.String())
+	str := endpoint.String()
+	if str != "" {
+		label = append(label, str)
 	}
 	return graphEscape(strings.Join(label, "\\n"))
 }

pkg/mesh/mesh.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build linux
 // +build linux
 
 package mesh
@@ -29,6 +30,8 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/vishvananda/netlink"
+	"golang.zx2c4.com/wireguard/wgctrl"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/squat/kilo/pkg/encapsulation"
 	"github.com/squat/kilo/pkg/iproute"
@@ -42,34 +45,34 @@ const (
 	kiloPath = "/var/lib/kilo"
 	// privateKeyPath is the filepath where the WireGuard private key is stored.
 	privateKeyPath = kiloPath + "/key"
-	// confPath is the filepath where the WireGuard configuration is stored.
-	confPath = kiloPath + "/conf"
 )
 
 // Mesh is able to create Kilo network meshes.
 type Mesh struct {
 	Backend
-	cleanUpIface bool
-	cni          bool
-	cniPath      string
-	enc          encapsulation.Encapsulator
-	externalIP   *net.IPNet
-	granularity  Granularity
-	hostname     string
-	internalIP   *net.IPNet
-	ipTables     *iptables.Controller
-	kiloIface    int
-	key          []byte
-	local        bool
-	port         uint32
-	priv         []byte
-	privIface    int
-	pub          []byte
-	resyncPeriod time.Duration
-	stop         chan struct{}
-	subnet       *net.IPNet
-	table        *route.Table
-	wireGuardIP  *net.IPNet
+	cleanUpIface        bool
+	cni                 bool
+	cniPath             string
+	enc                 encapsulation.Encapsulator
+	externalIP          *net.IPNet
+	granularity         Granularity
+	hostname            string
+	internalIP          *net.IPNet
+	ipTables            *iptables.Controller
+	kiloIface           int
+	kiloIfaceName       string
+	key                 []byte
+	local               bool
+	port                int
+	priv                wgtypes.Key
+	privIface           int
+	pub                 wgtypes.Key
+	resyncPeriod        time.Duration
+	iptablesForwardRule bool
+	stop                chan struct{}
+	subnet              *net.IPNet
+	table               *route.Table
+	wireGuardIP         *net.IPNet
 
 	// nodes and peers are mutable fields in the struct
 	// and need to be guarded.
@@ -86,23 +89,24 @@ type Mesh struct {
 }
 
 // New returns a new Mesh instance.
-func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, createIface bool, resyncPeriod time.Duration, logger log.Logger) (*Mesh, error) {
+func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, logger log.Logger) (*Mesh, error) {
 	if err := os.MkdirAll(kiloPath, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create directory to store configuration: %v", err)
 	}
-	private, err := ioutil.ReadFile(privateKeyPath)
-	private = bytes.Trim(private, "\n")
+	privateB, err := ioutil.ReadFile(privateKeyPath)
+	privateB = bytes.Trim(privateB, "\n")
+	private, err := wgtypes.ParseKey(string(privateB))
 	if err != nil {
 		level.Warn(logger).Log("msg", "no private key found on disk; generating one now")
-		if private, err = wireguard.GenKey(); err != nil {
+		if private, err = wgtypes.GeneratePrivateKey(); err != nil {
 			return nil, err
 		}
 	}
-	public, err := wireguard.PubKey(private)
+	public := private.PublicKey()
 	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(privateKeyPath, private, 0600); err != nil {
+	if err := ioutil.WriteFile(privateKeyPath, []byte(private.String()), 0600); err != nil {
 		return nil, fmt.Errorf("failed to write private key to disk: %v", err)
 	}
 	cniIndex, err := cniDeviceIndex()
@@ -111,7 +115,7 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 	}
 	var kiloIface int
 	if createIface {
-		kiloIface, _, err = wireguard.New(iface)
+		kiloIface, _, err = wireguard.New(iface, mtu)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create WireGuard interface: %v", err)
 		}
@@ -143,34 +147,42 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 		enc = encapsulation.Noop(enc.Strategy())
 		level.Debug(logger).Log("msg", "running without a private IP address")
 	}
+	var externalIP *net.IPNet
+	if prioritisePrivateAddr && privateIP != nil {
+		externalIP = privateIP
+	} else {
+		externalIP = publicIP
+	}
 	level.Debug(logger).Log("msg", fmt.Sprintf("using %s as the public IP address", publicIP.String()))
 	ipTables, err := iptables.New(iptables.WithLogger(log.With(logger, "component", "iptables")), iptables.WithResyncPeriod(resyncPeriod))
 	if err != nil {
 		return nil, fmt.Errorf("failed to IP tables controller: %v", err)
 	}
 	return &Mesh{
-		Backend:      backend,
-		cleanUpIface: cleanUpIface,
-		cni:          cni,
-		cniPath:      cniPath,
-		enc:          enc,
-		externalIP:   publicIP,
-		granularity:  granularity,
-		hostname:     hostname,
-		internalIP:   privateIP,
-		ipTables:     ipTables,
-		kiloIface:    kiloIface,
-		nodes:        make(map[string]*Node),
-		peers:        make(map[string]*Peer),
-		port:         port,
-		priv:         private,
-		privIface:    privIface,
-		pub:          public,
-		resyncPeriod: resyncPeriod,
-		local:        local,
-		stop:         make(chan struct{}),
-		subnet:       subnet,
-		table:        route.NewTable(),
+		Backend:             backend,
+		cleanUpIface:        cleanUpIface,
+		cni:                 cni,
+		cniPath:             cniPath,
+		enc:                 enc,
+		externalIP:          externalIP,
+		granularity:         granularity,
+		hostname:            hostname,
+		internalIP:          privateIP,
+		ipTables:            ipTables,
+		kiloIface:           kiloIface,
+		kiloIfaceName:       iface,
+		nodes:               make(map[string]*Node),
+		peers:               make(map[string]*Peer),
+		port:                port,
+		priv:                private,
+		privIface:           privIface,
+		pub:                 public,
+		resyncPeriod:        resyncPeriod,
+		iptablesForwardRule: iptablesForwardRule,
+		local:               local,
+		stop:                make(chan struct{}),
+		subnet:              subnet,
+		table:               route.NewTable(),
 		errorCounter: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Name: "kilo_errors_total",
 			Help: "Number of errors that occurred while administering the mesh.",
@@ -274,28 +286,23 @@ func (m *Mesh) syncNodes(e *NodeEvent) {
 	var diff bool
 	m.mu.Lock()
 	if !e.Node.Ready() {
-		level.Debug(logger).Log("msg", "received incomplete node", "node", e.Node)
-		// An existing node is no longer valid
-		// so remove it from the mesh.
-		if _, ok := m.nodes[e.Node.Name]; ok {
-			level.Info(logger).Log("msg", "node is no longer ready", "node", e.Node)
-			diff = true
-		}
-	} else {
-		switch e.Type {
-		case AddEvent:
-			fallthrough
-		case UpdateEvent:
-			if !nodesAreEqual(m.nodes[e.Node.Name], e.Node) {
-				diff = true
-			}
-			// Even if the nodes are the same,
-			// overwrite the old node to update the timestamp.
-			m.nodes[e.Node.Name] = e.Node
-		case DeleteEvent:
-			delete(m.nodes, e.Node.Name)
+		// Trace non ready nodes with their presence in the mesh.
+		_, ok := m.nodes[e.Node.Name]
+		level.Debug(logger).Log("msg", "received non ready node", "node", e.Node, "in-mesh", ok)
+	}
+	switch e.Type {
+	case AddEvent:
+		fallthrough
+	case UpdateEvent:
+		if !nodesAreEqual(m.nodes[e.Node.Name], e.Node) {
 			diff = true
 		}
+		// Even if the nodes are the same,
+		// overwrite the old node to update the timestamp.
+		m.nodes[e.Node.Name] = e.Node
+	case DeleteEvent:
+		delete(m.nodes, e.Node.Name)
+		diff = true
 	}
 	m.mu.Unlock()
 	if diff {
@@ -310,32 +317,27 @@ func (m *Mesh) syncPeers(e *PeerEvent) {
 	var diff bool
 	m.mu.Lock()
 	// Peers are indexed by public key.
-	key := string(e.Peer.PublicKey)
+	key := e.Peer.PublicKey.String()
 	if !e.Peer.Ready() {
-		level.Debug(logger).Log("msg", "received incomplete peer", "peer", e.Peer)
-		// An existing peer is no longer valid
-		// so remove it from the mesh.
-		if _, ok := m.peers[key]; ok {
-			level.Info(logger).Log("msg", "peer is no longer ready", "peer", e.Peer)
+		// Trace non ready peer with their presence in the mesh.
+		_, ok := m.peers[key]
+		level.Debug(logger).Log("msg", "received non ready peer", "peer", e.Peer, "in-mesh", ok)
+	}
+	switch e.Type {
+	case AddEvent:
+		fallthrough
+	case UpdateEvent:
+		if e.Old != nil && key != e.Old.PublicKey.String() {
+			delete(m.peers, e.Old.PublicKey.String())
 			diff = true
 		}
-	} else {
-		switch e.Type {
-		case AddEvent:
-			fallthrough
-		case UpdateEvent:
-			if e.Old != nil && key != string(e.Old.PublicKey) {
-				delete(m.peers, string(e.Old.PublicKey))
-				diff = true
-			}
-			if !peersAreEqual(m.peers[key], e.Peer) {
-				m.peers[key] = e.Peer
-				diff = true
-			}
-		case DeleteEvent:
-			delete(m.peers, key)
+		if !peersAreEqual(m.peers[key], e.Peer) {
+			m.peers[key] = e.Peer
 			diff = true
 		}
+	case DeleteEvent:
+		delete(m.peers, key)
+		diff = true
 	}
 	m.mu.Unlock()
 	if diff {
@@ -368,8 +370,10 @@ func (m *Mesh) checkIn() {
 
 func (m *Mesh) handleLocal(n *Node) {
 	// Allow the IPs to be overridden.
-	if n.Endpoint == nil || (n.Endpoint.DNS == "" && n.Endpoint.IP == nil) {
-		n.Endpoint = &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: m.externalIP.IP}, Port: m.port}
+	if !n.Endpoint.Ready() {
+		e := wireguard.NewEndpoint(m.externalIP.IP, m.port)
+		level.Info(m.logger).Log("msg", "overriding endpoint", "node", m.hostname, "old endpoint", n.Endpoint.String(), "new endpoint", e.String())
+		n.Endpoint = e
 	}
 	if n.InternalIP == nil && !n.NoInternalIP {
 		n.InternalIP = m.internalIP
@@ -389,6 +393,9 @@ func (m *Mesh) handleLocal(n *Node) {
 		PersistentKeepalive: n.PersistentKeepalive,
 		Subnet:              n.Subnet,
 		WireGuardIP:         m.wireGuardIP,
+		DiscoveredEndpoints: n.DiscoveredEndpoints,
+		AllowedLocationIPs:  n.AllowedLocationIPs,
+		Granularity:         m.granularity,
 	}
 	if !nodesAreEqual(n, local) {
 		level.Debug(m.logger).Log("msg", "local node differs from backend")
@@ -428,12 +435,12 @@ func (m *Mesh) applyTopology() {
 	nodes := make(map[string]*Node)
 	var readyNodes float64
 	for k := range m.nodes {
+		m.nodes[k].Granularity = m.granularity
 		if !m.nodes[k].Ready() {
 			continue
 		}
-		// Make a shallow copy of the node.
-		node := *m.nodes[k]
-		nodes[k] = &node
+		// Make it point to the node without copy.
+		nodes[k] = m.nodes[k]
 		readyNodes++
 	}
 	// Ensure only ready nodes are considered.
@@ -443,9 +450,8 @@ func (m *Mesh) applyTopology() {
 		if !m.peers[k].Ready() {
 			continue
 		}
-		// Make a shallow copy of the peer.
-		peer := *m.peers[k]
-		peers[k] = &peer
+		// Make it point the peer without copy.
+		peers[k] = m.peers[k]
 		readyPeers++
 	}
 	m.nodesGuage.Set(readyNodes)
@@ -461,16 +467,26 @@ func (m *Mesh) applyTopology() {
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	// Find the old configuration.
-	oldConfRaw, err := wireguard.ShowConf(link.Attrs().Name)
+
+	wgClient, err := wgctrl.New()
 	if err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	oldConf := wireguard.Parse(oldConfRaw)
-	updateNATEndpoints(nodes, peers, oldConf)
-	t, err := NewTopology(nodes, peers, m.granularity, m.hostname, nodes[m.hostname].Endpoint.Port, m.priv, m.subnet, nodes[m.hostname].PersistentKeepalive)
+	defer wgClient.Close()
+
+	// wgDevice is the current configuration of the wg interface.
+	wgDevice, err := wgClient.Device(m.kiloIfaceName)
+	if err != nil {
+		level.Error(m.logger).Log("error", err)
+		m.errorCounter.WithLabelValues("apply").Inc()
+		return
+	}
+
+	natEndpoints := discoverNATEndpoints(nodes, peers, wgDevice, m.logger)
+	nodes[m.hostname].DiscoveredEndpoints = natEndpoints
+	t, err := NewTopology(nodes, peers, m.granularity, m.hostname, nodes[m.hostname].Endpoint.Port(), m.priv, m.subnet, nodes[m.hostname].PersistentKeepalive, m.logger)
 	if err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
@@ -482,19 +498,8 @@ func (m *Mesh) applyTopology() {
 	} else {
 		m.wireGuardIP = nil
 	}
-	conf := t.Conf()
-	buf, err := conf.Bytes()
-	if err != nil {
-		level.Error(m.logger).Log("error", err)
-		m.errorCounter.WithLabelValues("apply").Inc()
-		return
-	}
-	if err := ioutil.WriteFile(confPath, buf, 0600); err != nil {
-		level.Error(m.logger).Log("error", err)
-		m.errorCounter.WithLabelValues("apply").Inc()
-		return
-	}
-	ipRules := t.Rules(m.cni)
+	ipRules := t.Rules(m.cni, m.iptablesForwardRule)
+
 	// If we are handling local routes, ensure the local
 	// tunnel has an IP address and IPIP traffic is allowed.
 	if m.enc.Strategy() != encapsulation.Never && m.local {
@@ -533,10 +538,12 @@ func (m *Mesh) applyTopology() {
 		}
 		// Setting the WireGuard configuration interrupts existing connections
 		// so only set the configuration if it has changed.
-		equal := conf.Equal(oldConf)
+		conf := t.Conf()
+		equal, diff := conf.Equal(wgDevice)
 		if !equal {
-			level.Info(m.logger).Log("msg", "WireGuard configurations are different")
-			if err := wireguard.SetConf(link.Attrs().Name, confPath); err != nil {
+			level.Info(m.logger).Log("msg", "WireGuard configurations are different", "diff", diff)
+			level.Debug(m.logger).Log("msg", "changing wg config", "config", conf.WGConfig())
+			if err := wgClient.ConfigureDevice(m.kiloIfaceName, conf.WGConfig()); err != nil {
 				level.Error(m.logger).Log("error", err)
 				m.errorCounter.WithLabelValues("apply").Inc()
 				return
@@ -591,10 +598,6 @@ func (m *Mesh) cleanUp() {
 		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up routes: %v", err))
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}
-	if err := os.Remove(confPath); err != nil {
-		level.Error(m.logger).Log("error", fmt.Sprintf("failed to delete configuration file: %v", err))
-		m.errorCounter.WithLabelValues("cleanUp").Inc()
-	}
 	if m.cleanUpIface {
 		if err := iproute.RemoveInterface(m.kiloIface); err != nil {
 			level.Error(m.logger).Log("error", fmt.Sprintf("failed to remove WireGuard interface: %v", err))
@@ -622,12 +625,8 @@ func (m *Mesh) resolveEndpoints() error {
 		if !m.nodes[k].Ready() {
 			continue
 		}
-		// If the node is ready, then the endpoint is not nil
-		// but it may not have a DNS name.
-		if m.nodes[k].Endpoint.DNS == "" {
-			continue
-		}
-		if err := resolveEndpoint(m.nodes[k].Endpoint); err != nil {
+		// Resolve the Endpoint
+		if _, err := m.nodes[k].Endpoint.UDPAddr(true); err != nil {
 			return err
 		}
 	}
@@ -638,33 +637,16 @@ func (m *Mesh) resolveEndpoints() error {
 			continue
 		}
 		// Peers may have nil endpoints.
-		if m.peers[k].Endpoint == nil || m.peers[k].Endpoint.DNS == "" {
+		if !m.peers[k].Endpoint.Ready() {
 			continue
 		}
-		if err := resolveEndpoint(m.peers[k].Endpoint); err != nil {
+		if _, err := m.peers[k].Endpoint.UDPAddr(true); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func resolveEndpoint(endpoint *wireguard.Endpoint) error {
-	ips, err := net.LookupIP(endpoint.DNS)
-	if err != nil {
-		return fmt.Errorf("failed to look up DNS name %q: %v", endpoint.DNS, err)
-	}
-	nets := make([]*net.IPNet, len(ips), len(ips))
-	for i := range ips {
-		nets[i] = oneAddressCIDR(ips[i])
-	}
-	sortIPs(nets)
-	if len(nets) == 0 {
-		return fmt.Errorf("did not find any addresses for DNS name %q", endpoint.DNS)
-	}
-	endpoint.IP = nets[0].IP
-	return nil
-}
-
 func isSelf(hostname string, node *Node) bool {
 	return node != nil && node.Name == hostname
 }
@@ -676,26 +658,26 @@ func nodesAreEqual(a, b *Node) bool {
 	if a == b {
 		return true
 	}
-	if !(a.Endpoint != nil) == (b.Endpoint != nil) {
+	// Check the DNS name first since this package
+	// is doing the DNS resolution.
+	if !a.Endpoint.Equal(b.Endpoint, true) {
 		return false
 	}
-	if a.Endpoint != nil {
-		if a.Endpoint.Port != b.Endpoint.Port {
-			return false
-		}
-		// Check the DNS name first since this package
-		// is doing the DNS resolution.
-		if a.Endpoint.DNS != b.Endpoint.DNS {
-			return false
-		}
-		if a.Endpoint.DNS == "" && !a.Endpoint.IP.Equal(b.Endpoint.IP) {
-			return false
-		}
-	}
 	// Ignore LastSeen when comparing equality we want to check if the nodes are
 	// equivalent. However, we do want to check if LastSeen has transitioned
 	// between valid and invalid.
-	return string(a.Key) == string(b.Key) && ipNetsEqual(a.WireGuardIP, b.WireGuardIP) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready() && a.PersistentKeepalive == b.PersistentKeepalive
+	return a.Key.String() == b.Key.String() &&
+		ipNetsEqual(a.WireGuardIP, b.WireGuardIP) &&
+		ipNetsEqual(a.InternalIP, b.InternalIP) &&
+		a.Leader == b.Leader &&
+		a.Location == b.Location &&
+		a.Name == b.Name &&
+		subnetsEqual(a.Subnet, b.Subnet) &&
+		a.Ready() == b.Ready() &&
+		a.PersistentKeepalive == b.PersistentKeepalive &&
+		discoveredEndpointsAreEqual(a.DiscoveredEndpoints, b.DiscoveredEndpoints) &&
+		ipNetSlicesEqual(a.AllowedLocationIPs, b.AllowedLocationIPs) &&
+		a.Granularity == b.Granularity
 }
 
 func peersAreEqual(a, b *Peer) bool {
@@ -705,31 +687,24 @@ func peersAreEqual(a, b *Peer) bool {
 	if a == b {
 		return true
 	}
-	if !(a.Endpoint != nil) == (b.Endpoint != nil) {
+	// Check the DNS name first since this package
+	// is doing the DNS resolution.
+	if !a.Endpoint.Equal(b.Endpoint, true) {
 		return false
 	}
-	if a.Endpoint != nil {
-		if a.Endpoint.Port != b.Endpoint.Port {
-			return false
-		}
-		// Check the DNS name first since this package
-		// is doing the DNS resolution.
-		if a.Endpoint.DNS != b.Endpoint.DNS {
-			return false
-		}
-		if a.Endpoint.DNS == "" && !a.Endpoint.IP.Equal(b.Endpoint.IP) {
-			return false
-		}
-	}
 	if len(a.AllowedIPs) != len(b.AllowedIPs) {
 		return false
 	}
 	for i := range a.AllowedIPs {
-		if !ipNetsEqual(a.AllowedIPs[i], b.AllowedIPs[i]) {
+		if !ipNetsEqual(&a.AllowedIPs[i], &b.AllowedIPs[i]) {
 			return false
 		}
 	}
-	return string(a.PublicKey) == string(b.PublicKey) && string(a.PresharedKey) == string(b.PresharedKey) && a.PersistentKeepalive == b.PersistentKeepalive
+	return a.PublicKey.String() == b.PublicKey.String() &&
+		(a.PresharedKey == nil) == (b.PresharedKey == nil) &&
+		(a.PresharedKey == nil || a.PresharedKey.String() == b.PresharedKey.String()) &&
+		(a.PersistentKeepaliveInterval == nil) == (b.PersistentKeepaliveInterval == nil) &&
+		(a.PersistentKeepaliveInterval == nil || *a.PersistentKeepaliveInterval == *b.PersistentKeepaliveInterval)
 }
 
 func ipNetsEqual(a, b *net.IPNet) bool {
@@ -745,6 +720,18 @@ func ipNetsEqual(a, b *net.IPNet) bool {
 	return a.IP.Equal(b.IP)
 }
 
+func ipNetSlicesEqual(a, b []net.IPNet) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if !ipNetsEqual(&a[i], &b[i]) {
+			return false
+		}
+	}
+	return true
+}
+
 func subnetsEqual(a, b *net.IPNet) bool {
 	if a == nil && b == nil {
 		return true
@@ -764,6 +751,37 @@ func subnetsEqual(a, b *net.IPNet) bool {
 	return true
 }
 
+func udpAddrsEqual(a, b *net.UDPAddr) bool {
+	if a == nil && b == nil {
+		return true
+	}
+	if (a != nil) != (b != nil) {
+		return false
+	}
+	if a.Zone != b.Zone {
+		return false
+	}
+	if a.Port != b.Port {
+		return false
+	}
+	return a.IP.Equal(b.IP)
+}
+
+func discoveredEndpointsAreEqual(a, b map[string]*net.UDPAddr) bool {
+	if a == nil && b == nil {
+		return true
+	}
+	if len(a) != len(b) {
+		return false
+	}
+	for k := range a {
+		if !udpAddrsEqual(a[k], b[k]) {
+			return false
+		}
+	}
+	return true
+}
+
 func linkByIndex(index int) (netlink.Link, error) {
 	link, err := netlink.LinkByIndex(index)
 	if err != nil {
@@ -772,21 +790,30 @@ func linkByIndex(index int) (netlink.Link, error) {
 	return link, nil
 }
 
-// updateNATEndpoints ensures that nodes and peers behind NAT update
-// their endpoints from the WireGuard configuration so they can roam.
-func updateNATEndpoints(nodes map[string]*Node, peers map[string]*Peer, conf *wireguard.Conf) {
-	keys := make(map[string]*wireguard.Peer)
+// discoverNATEndpoints uses the node's WireGuard configuration to returns a list of the most recently discovered endpoints for all nodes and peers behind NAT so that they can roam.
+// Discovered endpionts will never be DNS names, because WireGuard will always resolve them to net.UDPAddr.
+func discoverNATEndpoints(nodes map[string]*Node, peers map[string]*Peer, conf *wgtypes.Device, logger log.Logger) map[string]*net.UDPAddr {
+	natEndpoints := make(map[string]*net.UDPAddr)
+	keys := make(map[string]wgtypes.Peer)
 	for i := range conf.Peers {
-		keys[string(conf.Peers[i].PublicKey)] = conf.Peers[i]
+		keys[conf.Peers[i].PublicKey.String()] = conf.Peers[i]
 	}
 	for _, n := range nodes {
-		if peer, ok := keys[string(n.Key)]; ok && n.PersistentKeepalive > 0 {
-			n.Endpoint = peer.Endpoint
+		if peer, ok := keys[n.Key.String()]; ok && n.PersistentKeepalive != time.Duration(0) {
+			level.Debug(logger).Log("msg", "WireGuard Update NAT Endpoint", "node", n.Name, "endpoint", peer.Endpoint, "former-endpoint", n.Endpoint, "same", peer.Endpoint.String() == n.Endpoint.String(), "latest-handshake", peer.LastHandshakeTime)
+			// Don't update the endpoint, if there was never any handshake.
+			if !peer.LastHandshakeTime.Equal(time.Time{}) {
+				natEndpoints[n.Key.String()] = peer.Endpoint
+			}
 		}
 	}
 	for _, p := range peers {
-		if peer, ok := keys[string(p.PublicKey)]; ok && p.PersistentKeepalive > 0 {
-			p.Endpoint = peer.Endpoint
+		if peer, ok := keys[p.PublicKey.String()]; ok && p.PersistentKeepaliveInterval != nil {
+			if !peer.LastHandshakeTime.Equal(time.Time{}) {
+				natEndpoints[p.PublicKey.String()] = peer.Endpoint
+			}
 		}
 	}
+	level.Debug(logger).Log("msg", "Discovered WireGuard NAT Endpoints", "DiscoveredEndpoints", natEndpoints)
+	return natEndpoints
 }

pkg/mesh/mesh_test.go

@@ -19,9 +19,21 @@ import (
 	"testing"
 	"time"
 
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
+func mustKey() wgtypes.Key {
+	if k, err := wgtypes.GeneratePrivateKey(); err != nil {
+		panic(err.Error())
+	} else {
+		return k
+	}
+}
+
+var key = mustKey()
+
 func TestReady(t *testing.T) {
 	internalIP := oneAddressCIDR(net.ParseIP("1.1.1.1"))
 	externalIP := oneAddressCIDR(net.ParseIP("2.2.2.2"))
@@ -44,7 +56,7 @@ func TestReady(t *testing.T) {
 			name: "empty endpoint",
 			node: &Node{
 				InternalIP: internalIP,
-				Key:        []byte{},
+				Key:        key,
 				Subnet:     &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(16, 32)},
 			},
 			ready: false,
@@ -52,9 +64,9 @@ func TestReady(t *testing.T) {
 		{
 			name: "empty endpoint IP",
 			node: &Node{
-				Endpoint:   &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{}, Port: DefaultKiloPort},
+				Endpoint:   wireguard.NewEndpoint(nil, DefaultKiloPort),
 				InternalIP: internalIP,
-				Key:        []byte{},
+				Key:        wgtypes.Key{},
 				Subnet:     &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(16, 32)},
 			},
 			ready: false,
@@ -62,9 +74,9 @@ func TestReady(t *testing.T) {
 		{
 			name: "empty endpoint port",
 			node: &Node{
-				Endpoint:   &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: externalIP.IP}},
+				Endpoint:   wireguard.NewEndpoint(externalIP.IP, 0),
 				InternalIP: internalIP,
-				Key:        []byte{},
+				Key:        wgtypes.Key{},
 				Subnet:     &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(16, 32)},
 			},
 			ready: false,
@@ -72,8 +84,8 @@ func TestReady(t *testing.T) {
 		{
 			name: "empty internal IP",
 			node: &Node{
-				Endpoint: &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: externalIP.IP}, Port: DefaultKiloPort},
-				Key:      []byte{},
+				Endpoint: wireguard.NewEndpoint(externalIP.IP, DefaultKiloPort),
+				Key:      wgtypes.Key{},
 				Subnet:   &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(16, 32)},
 			},
 			ready: false,
@@ -81,7 +93,7 @@ func TestReady(t *testing.T) {
 		{
 			name: "empty key",
 			node: &Node{
-				Endpoint:   &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: externalIP.IP}, Port: DefaultKiloPort},
+				Endpoint:   wireguard.NewEndpoint(externalIP.IP, DefaultKiloPort),
 				InternalIP: internalIP,
 				Subnet:     &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(16, 32)},
 			},
@@ -90,18 +102,18 @@ func TestReady(t *testing.T) {
 		{
 			name: "empty subnet",
 			node: &Node{
-				Endpoint:   &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: externalIP.IP}, Port: DefaultKiloPort},
+				Endpoint:   wireguard.NewEndpoint(externalIP.IP, DefaultKiloPort),
 				InternalIP: internalIP,
-				Key:        []byte{},
+				Key:        wgtypes.Key{},
 			},
 			ready: false,
 		},
 		{
 			name: "valid",
 			node: &Node{
-				Endpoint:   &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: externalIP.IP}, Port: DefaultKiloPort},
+				Endpoint:   wireguard.NewEndpoint(externalIP.IP, DefaultKiloPort),
 				InternalIP: internalIP,
-				Key:        []byte{},
+				Key:        key,
 				LastSeen:   time.Now().Unix(),
 				Subnet:     &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(16, 32)},
 			},

pkg/mesh/routes.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build linux
 // +build linux
 
 package mesh
@@ -39,7 +40,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 		var gw net.IP
 		for _, segment := range t.segments {
 			if segment.location == t.location {
-				gw = enc.Gw(segment.endpoint.IP, segment.privateIPs[segment.leader], segment.cidrs[segment.leader])
+				gw = enc.Gw(t.updateEndpoint(segment.endpoint, segment.key, &segment.persistentKeepalive).IP(), segment.privateIPs[segment.leader], segment.cidrs[segment.leader])
 				break
 			}
 		}
@@ -108,12 +109,23 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 					Protocol:  unix.RTPROT_STATIC,
 				}, enc.Strategy(), t.privateIP, tunlIface))
 			}
+			// For segments / locations other than the location of this instance of kg,
+			// we need to set routes for allowed location IPs over the leader in the current location.
+			for i := range segment.allowedLocationIPs {
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       &segment.allowedLocationIPs[i],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
 		}
 		// Add routes for the allowed IPs of peers.
 		for _, peer := range t.peers {
 			for i := range peer.AllowedIPs {
 				routes = append(routes, encapsulateRoute(&netlink.Route{
-					Dst:       peer.AllowedIPs[i],
+					Dst:       &peer.AllowedIPs[i],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        gw,
 					LinkIndex: privIface,
@@ -184,7 +196,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 			// equals the external IP. This means that the node
 			// is only accessible through an external IP and we
 			// cannot encapsulate traffic to an IP through the IP.
-			if segment.privateIPs == nil || segment.privateIPs[i].Equal(segment.endpoint.IP) {
+			if segment.privateIPs == nil || segment.privateIPs[i].Equal(t.updateEndpoint(segment.endpoint, segment.key, &segment.persistentKeepalive).IP()) {
 				continue
 			}
 			// Add routes to the private IPs of nodes in other segments.
@@ -198,17 +210,96 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 				Protocol:  unix.RTPROT_STATIC,
 			})
 		}
-	}
-	// Add routes for the allowed IPs of peers.
-	for _, peer := range t.peers {
-		for i := range peer.AllowedIPs {
+		// For segments / locations other than the location of this instance of kg,
+		// we need to set routes for allowed location IPs over the wg interface.
+		for i := range segment.allowedLocationIPs {
 			routes = append(routes, &netlink.Route{
-				Dst:       peer.AllowedIPs[i],
+				Dst:       &segment.allowedLocationIPs[i],
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
 				LinkIndex: kiloIface,
 				Protocol:  unix.RTPROT_STATIC,
 			})
 		}
 	}
+	// Add routes for the allowed IPs of peers.
+	for _, peer := range t.peers {
+		for i := range peer.AllowedIPs {
+			routes = append(routes, &netlink.Route{
+				Dst:       &peer.AllowedIPs[i],
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+	}
+	return routes, rules
+}
+
+// PeerRoutes generates a slice of routes and rules for a given peer in the Topology.
+func (t *Topology) PeerRoutes(name string, kiloIface int, additionalAllowedIPs []net.IPNet) ([]*netlink.Route, []*netlink.Rule) {
+	var routes []*netlink.Route
+	var rules []*netlink.Rule
+	for _, segment := range t.segments {
+		for i := range segment.cidrs {
+			// Add routes to the Pod CIDRs of nodes in other segments.
+			routes = append(routes, &netlink.Route{
+				Dst:       segment.cidrs[i],
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+		for i := range segment.privateIPs {
+			// Add routes to the private IPs of nodes in other segments.
+			routes = append(routes, &netlink.Route{
+				Dst:       oneAddressCIDR(segment.privateIPs[i]),
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+		// Add routes for the allowed location IPs of all segments.
+		for i := range segment.allowedLocationIPs {
+			routes = append(routes, &netlink.Route{
+				Dst:       &segment.allowedLocationIPs[i],
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+		routes = append(routes, &netlink.Route{
+			Dst:       oneAddressCIDR(segment.wireGuardIP),
+			LinkIndex: kiloIface,
+			Protocol:  unix.RTPROT_STATIC,
+		})
+	}
+	// Add routes for the allowed IPs of peers.
+	for _, peer := range t.peers {
+		// Don't add routes to ourselves.
+		if peer.Name == name {
+			continue
+		}
+		for i := range peer.AllowedIPs {
+			routes = append(routes, &netlink.Route{
+				Dst:       &peer.AllowedIPs[i],
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+	}
+	for i := range additionalAllowedIPs {
+		routes = append(routes, &netlink.Route{
+			Dst:       &additionalAllowedIPs[i],
+			Flags:     int(netlink.FLAG_ONLINK),
+			Gw:        t.segments[0].wireGuardIP,
+			LinkIndex: kiloIface,
+			Protocol:  unix.RTPROT_STATIC,
+		})
+	}
+
 	return routes, rules
 }
 
@@ -220,24 +311,62 @@ func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy,
 }
 
 // Rules returns the iptables rules required by the local node.
-func (t *Topology) Rules(cni bool) []iptables.Rule {
+func (t *Topology) Rules(cni, iptablesForwardRule bool) []iptables.Rule {
 	var rules []iptables.Rule
 	rules = append(rules, iptables.NewIPv4Chain("nat", "KILO-NAT"))
 	rules = append(rules, iptables.NewIPv6Chain("nat", "KILO-NAT"))
 	if cni {
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(t.subnet.IP)), "nat", "POSTROUTING", "-s", t.subnet.String(), "-m", "comment", "--comment", "Kilo: jump to KILO-NAT chain", "-j", "KILO-NAT"))
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(t.subnet.IP), "nat", "POSTROUTING", "-s", t.subnet.String(), "-m", "comment", "--comment", "Kilo: jump to KILO-NAT chain", "-j", "KILO-NAT"))
+		// Some linux distros or docker will set forward DROP in the filter table.
+		// To still be able to have pod to pod communication we need to ALLOW packets from and to pod CIDRs within a location.
+		// Leader nodes will forward packets from all nodes within a location because they act as a gateway for them.
+		// Non leader nodes only need to allow packages from and to their own pod CIDR.
+		if iptablesForwardRule && t.leader {
+			for _, s := range t.segments {
+				if s.location == t.location {
+					// Make sure packets to and from pod cidrs are not dropped in the forward chain.
+					for _, c := range s.cidrs {
+						rules = append(rules, iptables.NewRule(iptables.GetProtocol(c.IP), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets from the pod subnet", "-s", c.String(), "-j", "ACCEPT"))
+						rules = append(rules, iptables.NewRule(iptables.GetProtocol(c.IP), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets to the pod subnet", "-d", c.String(), "-j", "ACCEPT"))
+					}
+					// Make sure packets to and from allowed location IPs are not dropped in the forward chain.
+					for _, c := range s.allowedLocationIPs {
+						rules = append(rules, iptables.NewRule(iptables.GetProtocol(c.IP), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets from allowed location IPs", "-s", c.String(), "-j", "ACCEPT"))
+						rules = append(rules, iptables.NewRule(iptables.GetProtocol(c.IP), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets to allowed location IPs", "-d", c.String(), "-j", "ACCEPT"))
+					}
+					// Make sure packets to and from private IPs are not dropped in the forward chain.
+					for _, c := range s.privateIPs {
+						rules = append(rules, iptables.NewRule(iptables.GetProtocol(c), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets from private IPs", "-s", oneAddressCIDR(c).String(), "-j", "ACCEPT"))
+						rules = append(rules, iptables.NewRule(iptables.GetProtocol(c), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets to private IPs", "-d", oneAddressCIDR(c).String(), "-j", "ACCEPT"))
+					}
+				}
+			}
+		} else if iptablesForwardRule {
+			rules = append(rules, iptables.NewRule(iptables.GetProtocol(t.subnet.IP), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets from the node's pod subnet", "-s", t.subnet.String(), "-j", "ACCEPT"))
+			rules = append(rules, iptables.NewRule(iptables.GetProtocol(t.subnet.IP), "filter", "FORWARD", "-m", "comment", "--comment", "Kilo: forward packets to the node's pod subnet", "-d", t.subnet.String(), "-j", "ACCEPT"))
+		}
 	}
 	for _, s := range t.segments {
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(s.wireGuardIP)), "nat", "KILO-NAT", "-d", oneAddressCIDR(s.wireGuardIP).String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuared IPs", "-j", "RETURN"))
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(s.wireGuardIP), "nat", "KILO-NAT", "-d", oneAddressCIDR(s.wireGuardIP).String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuared IPs", "-j", "RETURN"))
 		for _, aip := range s.allowedIPs {
-			rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-j", "RETURN"))
+			rules = append(rules, iptables.NewRule(iptables.GetProtocol(aip.IP), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-j", "RETURN"))
+		}
+		// Make sure packets to allowed location IPs go through the KILO-NAT chain, so they can be MASQUERADEd,
+		// Otherwise packets to these destinations will reach the destination, but never find their way back.
+		// We only want to NAT in locations of the corresponding allowed location IPs.
+		if t.location == s.location {
+			for _, alip := range s.allowedLocationIPs {
+				rules = append(rules,
+					iptables.NewRule(iptables.GetProtocol(alip.IP), "nat", "POSTROUTING", "-d", alip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"),
+				)
+			}
 		}
 	}
 	for _, p := range t.peers {
 		for _, aip := range p.AllowedIPs {
 			rules = append(rules,
-				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "POSTROUTING", "-s", aip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"),
-				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for peers", "-j", "RETURN"),
+				iptables.NewRule(iptables.GetProtocol(aip.IP), "nat", "POSTROUTING", "-s", aip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"),
+				iptables.NewRule(iptables.GetProtocol(aip.IP), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for peers", "-j", "RETURN"),
 			)
 		}
 	}

pkg/mesh/routes_test.go

@@ -74,6 +74,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[2].cidrs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -82,17 +89,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -125,17 +132,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -189,21 +196,21 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: privIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: privIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: privIface,
@@ -259,17 +266,24 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["d"].Name).segments[1].wireGuardIP,
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[1],
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
+				{
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -294,6 +308,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].cidrs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -316,17 +337,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -373,17 +394,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -422,6 +443,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[3].cidrs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -430,17 +458,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -480,6 +508,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["d"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -488,17 +523,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -538,6 +573,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["d"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -546,17 +588,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -597,17 +639,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -656,17 +698,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -740,21 +782,21 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: privIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: privIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: privIface,
@@ -826,21 +868,21 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: tunlIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: tunlIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
 					Gw:        nodes["b"].InternalIP.IP,
 					LinkIndex: tunlIface,
@@ -875,6 +917,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["c"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -897,17 +946,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -955,17 +1004,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
@@ -1005,6 +1054,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       &nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["d"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -1013,17 +1069,17 @@ func TestRoutes(t *testing.T) {
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[0],
+					Dst:       &peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["a"].AllowedIPs[1],
+					Dst:       &peers["a"].AllowedIPs[1],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
 				{
-					Dst:       peers["b"].AllowedIPs[0],
+					Dst:       &peers["b"].AllowedIPs[0],
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},

pkg/mesh/topology.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,6 +18,11 @@ import (
 	"errors"
 	"net"
 	"sort"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/squat/kilo/pkg/wireguard"
 )
@@ -30,8 +35,8 @@ const (
 // Topology represents the logical structure of the overlay network.
 type Topology struct {
 	// key is the private key of the node creating the topology.
-	key  []byte
-	port uint32
+	key  wgtypes.Key
+	port int
 	// Location is the logical location of the local host.
 	location string
 	segments []*segment
@@ -44,7 +49,7 @@ type Topology struct {
 	leader bool
 	// persistentKeepalive is the interval in seconds of the emission
 	// of keepalive packets by the local node to its peers.
-	persistentKeepalive int
+	persistentKeepalive time.Duration
 	// privateIP is the private IP address of the local node.
 	privateIP *net.IPNet
 	// subnet is the Pod subnet of the local node.
@@ -55,12 +60,17 @@ type Topology struct {
 	// the IP is the 0th address in the subnet, i.e. the CIDR
 	// is equal to the Kilo subnet.
 	wireGuardCIDR *net.IPNet
+	// discoveredEndpoints is the updated map of valid discovered Endpoints
+	discoveredEndpoints map[string]*net.UDPAddr
+	logger              log.Logger
 }
 
+// segment represents one logical unit in the topology that is united by one common WireGuard IP.
 type segment struct {
-	allowedIPs []*net.IPNet
-	endpoint   *wireguard.Endpoint
-	key        []byte
+	allowedIPs          []net.IPNet
+	endpoint            *wireguard.Endpoint
+	key                 wgtypes.Key
+	persistentKeepalive time.Duration
 	// Location is the logical location of this segment.
 	location string
 
@@ -75,10 +85,17 @@ type segment struct {
 	// wireGuardIP is the allocated IP address of the WireGuard
 	// interface on the leader of the segment.
 	wireGuardIP net.IP
+	// allowedLocationIPs are not part of the cluster and are not peers.
+	// They are directly routable from nodes within the segment.
+	// A classic example is a printer that ought to be routable from other locations.
+	allowedLocationIPs []net.IPNet
 }
 
 // NewTopology creates a new Topology struct from a given set of nodes and peers.
-func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port uint32, key []byte, subnet *net.IPNet, persistentKeepalive int) (*Topology, error) {
+func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port int, key wgtypes.Key, subnet *net.IPNet, persistentKeepalive time.Duration, logger log.Logger) (*Topology, error) {
+	if logger == nil {
+		logger = log.NewNopLogger()
+	}
 	topoMap := make(map[string][]*Node)
 	for _, node := range nodes {
 		var location string
@@ -106,7 +123,18 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 		localLocation = nodeLocationPrefix + hostname
 	}
 
-	t := Topology{key: key, port: port, hostname: hostname, location: localLocation, persistentKeepalive: persistentKeepalive, privateIP: nodes[hostname].InternalIP, subnet: nodes[hostname].Subnet, wireGuardCIDR: subnet}
+	t := Topology{
+		key:                 key,
+		port:                port,
+		hostname:            hostname,
+		location:            localLocation,
+		persistentKeepalive: persistentKeepalive,
+		privateIP:           nodes[hostname].InternalIP,
+		subnet:              nodes[hostname].Subnet,
+		wireGuardCIDR:       subnet,
+		discoveredEndpoints: make(map[string]*net.UDPAddr),
+		logger:              logger,
+	}
 	for location := range topoMap {
 		// Sort the location so the result is stable.
 		sort.Slice(topoMap[location], func(i, j int) bool {
@@ -116,7 +144,9 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 		if location == localLocation && topoMap[location][leader].Name == hostname {
 			t.leader = true
 		}
-		var allowedIPs []*net.IPNet
+		var allowedIPs []net.IPNet
+		allowedLocationIPsMap := make(map[string]struct{})
+		var allowedLocationIPs []net.IPNet
 		var cidrs []*net.IPNet
 		var hostnames []string
 		var privateIPs []net.IP
@@ -125,24 +155,41 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			// - the node's allocated subnet
 			// - the node's WireGuard IP
 			// - the node's internal IP
-			allowedIPs = append(allowedIPs, node.Subnet)
+			// - IPs that were specified by the allowed-location-ips annotation
+			if node.Subnet != nil {
+				allowedIPs = append(allowedIPs, *node.Subnet)
+			}
+			for _, ip := range node.AllowedLocationIPs {
+				if _, ok := allowedLocationIPsMap[ip.String()]; !ok {
+					allowedLocationIPs = append(allowedLocationIPs, ip)
+					allowedLocationIPsMap[ip.String()] = struct{}{}
+				}
+			}
 			if node.InternalIP != nil {
-				allowedIPs = append(allowedIPs, oneAddressCIDR(node.InternalIP.IP))
+				allowedIPs = append(allowedIPs, *oneAddressCIDR(node.InternalIP.IP))
 				privateIPs = append(privateIPs, node.InternalIP.IP)
 			}
 			cidrs = append(cidrs, node.Subnet)
 			hostnames = append(hostnames, node.Name)
 		}
-		t.segments = append(t.segments, &segment{
-			allowedIPs: allowedIPs,
-			endpoint:   topoMap[location][leader].Endpoint,
-			key:        topoMap[location][leader].Key,
-			location:   location,
-			cidrs:      cidrs,
-			hostnames:  hostnames,
-			leader:     leader,
-			privateIPs: privateIPs,
+		// The sorting has no function, but makes testing easier.
+		sort.Slice(allowedLocationIPs, func(i, j int) bool {
+			return allowedLocationIPs[i].String() < allowedLocationIPs[j].String()
 		})
+		t.segments = append(t.segments, &segment{
+			allowedIPs:          allowedIPs,
+			endpoint:            topoMap[location][leader].Endpoint,
+			key:                 topoMap[location][leader].Key,
+			persistentKeepalive: topoMap[location][leader].PersistentKeepalive,
+			location:            location,
+			cidrs:               cidrs,
+			hostnames:           hostnames,
+			leader:              leader,
+			privateIPs:          privateIPs,
+			allowedLocationIPs:  allowedLocationIPs,
+		})
+		level.Debug(t.logger).Log("msg", "generated segment", "location", location, "allowedIPs", allowedIPs, "endpoint", topoMap[location][leader].Endpoint, "cidrs", cidrs, "hostnames", hostnames, "leader", leader, "privateIPs", privateIPs, "allowedLocationIPs", allowedLocationIPs)
+
 	}
 	// Sort the Topology segments so the result is stable.
 	sort.Slice(t.segments, func(i, j int) bool {
@@ -159,6 +206,10 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 	// We need to defensively deduplicate peer allowed IPs. If two peers claim the same IP,
 	// the WireGuard configuration could flap, causing the interface to churn.
 	t.peers = deduplicatePeerIPs(t.peers)
+	// Copy the host node DiscoveredEndpoints in the topology as a starting point.
+	for key := range nodes[hostname].DiscoveredEndpoints {
+		t.discoveredEndpoints[key] = nodes[hostname].DiscoveredEndpoints[key]
+	}
 	// Allocate IPs to the segment leaders in a stable, coordination-free manner.
 	a := newAllocator(*subnet)
 	for _, segment := range t.segments {
@@ -167,42 +218,120 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			return nil, errors.New("failed to allocate an IP address; ran out of IP addresses")
 		}
 		segment.wireGuardIP = ipNet.IP
-		segment.allowedIPs = append(segment.allowedIPs, oneAddressCIDR(ipNet.IP))
+		segment.allowedIPs = append(segment.allowedIPs, *oneAddressCIDR(ipNet.IP))
 		if t.leader && segment.location == t.location {
 			t.wireGuardCIDR = &net.IPNet{IP: ipNet.IP, Mask: subnet.Mask}
 		}
+
+		// Now that the topology is ordered, update the discoveredEndpoints map
+		// add new ones by going through the ordered topology: segments, nodes
+		for _, node := range topoMap[segment.location] {
+			for key := range node.DiscoveredEndpoints {
+				if _, ok := t.discoveredEndpoints[key]; !ok {
+					t.discoveredEndpoints[key] = node.DiscoveredEndpoints[key]
+				}
+			}
+		}
+		// Check for intersecting IPs in allowed location IPs
+		segment.allowedLocationIPs = t.filterAllowedLocationIPs(segment.allowedLocationIPs, segment.location)
 	}
 
+	level.Debug(t.logger).Log("msg", "generated topology", "location", t.location, "hostname", t.hostname, "wireGuardIP", t.wireGuardCIDR, "privateIP", t.privateIP, "subnet", t.subnet, "leader", t.leader)
 	return &t, nil
 }
 
+func intersect(n1, n2 net.IPNet) bool {
+	return n1.Contains(n2.IP) || n2.Contains(n1.IP)
+}
+
+func (t *Topology) filterAllowedLocationIPs(ips []net.IPNet, location string) (ret []net.IPNet) {
+CheckIPs:
+	for _, ip := range ips {
+		for _, s := range t.segments {
+			// Check if allowed location IPs are also allowed in other locations.
+			if location != s.location {
+				for _, i := range s.allowedLocationIPs {
+					if intersect(ip, i) {
+						level.Warn(t.logger).Log("msg", "overlapping allowed location IPnets", "IP", ip.String(), "IP2", i.String(), "segment-location", s.location)
+						continue CheckIPs
+					}
+				}
+			}
+			// Check if allowed location IPs intersect with the allowed IPs.
+			for _, i := range s.allowedIPs {
+				if intersect(ip, i) {
+					level.Warn(t.logger).Log("msg", "overlapping allowed location IPnet with allowed IPnets", "IP", ip.String(), "IP2", i.String(), "segment-location", s.location)
+					continue CheckIPs
+				}
+			}
+			// Check if allowed location IPs intersect with the private IPs of the segment.
+			for _, i := range s.privateIPs {
+				if ip.Contains(i) {
+					level.Warn(t.logger).Log("msg", "overlapping allowed location IPnet with privateIP", "IP", ip.String(), "IP2", i.String(), "segment-location", s.location)
+					continue CheckIPs
+				}
+			}
+		}
+		// Check if allowed location IPs intersect with allowed IPs of peers.
+		for _, p := range t.peers {
+			for _, i := range p.AllowedIPs {
+				if intersect(ip, i) {
+					level.Warn(t.logger).Log("msg", "overlapping allowed location IPnet with peer IPnet", "IP", ip.String(), "IP2", i.String(), "peer", p.Name)
+					continue CheckIPs
+				}
+			}
+		}
+		ret = append(ret, ip)
+	}
+	return
+}
+
+func (t *Topology) updateEndpoint(endpoint *wireguard.Endpoint, key wgtypes.Key, persistentKeepalive *time.Duration) *wireguard.Endpoint {
+	// Do not update non-nat peers
+	if persistentKeepalive == nil || *persistentKeepalive == time.Duration(0) {
+		return endpoint
+	}
+	e, ok := t.discoveredEndpoints[key.String()]
+	if ok {
+		return wireguard.NewEndpointFromUDPAddr(e)
+	}
+	return endpoint
+}
+
 // Conf generates a WireGuard configuration file for a given Topology.
 func (t *Topology) Conf() *wireguard.Conf {
 	c := &wireguard.Conf{
-		Interface: &wireguard.Interface{
-			PrivateKey: t.key,
-			ListenPort: t.port,
+		Config: wgtypes.Config{
+			PrivateKey:   &t.key,
+			ListenPort:   &t.port,
+			ReplacePeers: true,
 		},
 	}
 	for _, s := range t.segments {
 		if s.location == t.location {
 			continue
 		}
-		peer := &wireguard.Peer{
-			AllowedIPs:          s.allowedIPs,
-			Endpoint:            s.endpoint,
-			PersistentKeepalive: t.persistentKeepalive,
-			PublicKey:           s.key,
+		peer := wireguard.Peer{
+			PeerConfig: wgtypes.PeerConfig{
+				AllowedIPs:                  append(s.allowedIPs, s.allowedLocationIPs...),
+				PersistentKeepaliveInterval: &t.persistentKeepalive,
+				PublicKey:                   s.key,
+				ReplaceAllowedIPs:           true,
+			},
+			Endpoint: t.updateEndpoint(s.endpoint, s.key, &s.persistentKeepalive),
 		}
 		c.Peers = append(c.Peers, peer)
 	}
 	for _, p := range t.peers {
-		peer := &wireguard.Peer{
-			AllowedIPs:          p.AllowedIPs,
-			Endpoint:            p.Endpoint,
-			PersistentKeepalive: t.persistentKeepalive,
-			PresharedKey:        p.PresharedKey,
-			PublicKey:           p.PublicKey,
+		peer := wireguard.Peer{
+			PeerConfig: wgtypes.PeerConfig{
+				AllowedIPs:                  p.AllowedIPs,
+				PersistentKeepaliveInterval: &t.persistentKeepalive,
+				PresharedKey:                p.PresharedKey,
+				PublicKey:                   p.PublicKey,
+				ReplaceAllowedIPs:           true,
+			},
+			Endpoint: t.updateEndpoint(p.Endpoint, p.PublicKey, p.PersistentKeepaliveInterval),
 		}
 		c.Peers = append(c.Peers, peer)
 	}
@@ -216,34 +345,39 @@ func (t *Topology) AsPeer() *wireguard.Peer {
 		if s.location != t.location {
 			continue
 		}
-		return &wireguard.Peer{
-			AllowedIPs: s.allowedIPs,
-			Endpoint:   s.endpoint,
-			PublicKey:  s.key,
+		p := &wireguard.Peer{
+			PeerConfig: wgtypes.PeerConfig{
+				AllowedIPs: s.allowedIPs,
+				PublicKey:  s.key,
+			},
+			Endpoint: s.endpoint,
 		}
+		return p
 	}
 	return nil
 }
 
 // PeerConf generates a WireGuard configuration file for a given peer in a Topology.
 func (t *Topology) PeerConf(name string) *wireguard.Conf {
-	var pka int
-	var psk []byte
+	var pka *time.Duration
+	var psk *wgtypes.Key
 	for i := range t.peers {
 		if t.peers[i].Name == name {
-			pka = t.peers[i].PersistentKeepalive
+			pka = t.peers[i].PersistentKeepaliveInterval
 			psk = t.peers[i].PresharedKey
 			break
 		}
 	}
 	c := &wireguard.Conf{}
 	for _, s := range t.segments {
-		peer := &wireguard.Peer{
-			AllowedIPs:          s.allowedIPs,
-			Endpoint:            s.endpoint,
-			PersistentKeepalive: pka,
-			PresharedKey:        psk,
-			PublicKey:           s.key,
+		peer := wireguard.Peer{
+			PeerConfig: wgtypes.PeerConfig{
+				AllowedIPs:                  append(s.allowedIPs, s.allowedLocationIPs...),
+				PersistentKeepaliveInterval: pka,
+				PresharedKey:                psk,
+				PublicKey:                   s.key,
+			},
+			Endpoint: t.updateEndpoint(s.endpoint, s.key, &s.persistentKeepalive),
 		}
 		c.Peers = append(c.Peers, peer)
 	}
@@ -251,11 +385,13 @@ func (t *Topology) PeerConf(name string) *wireguard.Conf {
 		if t.peers[i].Name == name {
 			continue
 		}
-		peer := &wireguard.Peer{
-			AllowedIPs:          t.peers[i].AllowedIPs,
-			PersistentKeepalive: pka,
-			PublicKey:           t.peers[i].PublicKey,
-			Endpoint:            t.peers[i].Endpoint,
+		peer := wireguard.Peer{
+			PeerConfig: wgtypes.PeerConfig{
+				AllowedIPs:                  t.peers[i].AllowedIPs,
+				PersistentKeepaliveInterval: pka,
+				PublicKey:                   t.peers[i].PublicKey,
+			},
+			Endpoint: t.updateEndpoint(t.peers[i].Endpoint, t.peers[i].PublicKey, t.peers[i].PersistentKeepaliveInterval),
 		}
 		c.Peers = append(c.Peers, peer)
 	}
@@ -276,13 +412,13 @@ func findLeader(nodes []*Node) int {
 	var leaders, public []int
 	for i := range nodes {
 		if nodes[i].Leader {
-			if isPublic(nodes[i].Endpoint.IP) {
+			if isPublic(nodes[i].Endpoint.IP()) {
 				return i
 			}
 			leaders = append(leaders, i)
 
 		}
-		if isPublic(nodes[i].Endpoint.IP) {
+		if nodes[i].Endpoint.IP() != nil && isPublic(nodes[i].Endpoint.IP()) {
 			public = append(public, i)
 		}
 	}
@@ -302,10 +438,12 @@ func deduplicatePeerIPs(peers []*Peer) []*Peer {
 		p := Peer{
 			Name: peer.Name,
 			Peer: wireguard.Peer{
-				Endpoint:            peer.Endpoint,
-				PersistentKeepalive: peer.PersistentKeepalive,
-				PresharedKey:        peer.PresharedKey,
-				PublicKey:           peer.PublicKey,
+				PeerConfig: wgtypes.PeerConfig{
+					PersistentKeepaliveInterval: peer.PersistentKeepaliveInterval,
+					PresharedKey:                peer.PresharedKey,
+					PublicKey:                   peer.PublicKey,
+				},
+				Endpoint: peer.Endpoint,
 			},
 		}
 		for _, ip := range peer.AllowedIPs {