Merge pull request #225 from squat/fix_scope

pkg/k8s: fix resource scope of Kilo CRD

.github/workflows/ci.yml

@@ -0,0 +1,157 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+    tags:
+      - "*"
+  pull_request:
+    branches: [ main ]
+  schedule:
+  - cron:  '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Build
+      run: make
+
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Build kg and kgctl for all Linux Architectures
+      run: make all-build
+
+  darwin:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Build kgctl for Darwin amd64
+      run: make OS=darwin ARCH=amd64
+    - name: Build kgctl for Darwin arm64
+      run: make OS=darwin ARCH=arm64
+
+  windows:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Build kgctl for Windows
+      run: make OS=windows
+
+  unit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Run Unit Tests
+      run: make unit
+
+  e2e:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Run e2e Tests
+      run: make e2e
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Lint Code
+      run: make lint
+
+  container:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Enable Experimental Docker CLI
+      run: |
+        echo $'{\n  "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+        mkdir -p ~/.docker
+        echo $'{\n  "experimental": "enabled"\n}' | sudo tee ~/.docker/config.json
+        sudo service docker restart
+        docker version -f '{{.Client.Experimental}}'
+        docker version -f '{{.Server.Experimental}}'
+        docker buildx version
+    - name: Container
+      run: make container
+
+  push:
+    if: github.event_name != 'pull_request'
+    needs:
+    - build
+    - linux
+    - darwin
+    - windows
+    - unit
+    - lint
+    - container
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Enable Experimental Docker CLI
+      run: |
+        echo $'{\n  "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+        mkdir -p ~/.docker
+        echo $'{\n  "experimental": "enabled"\n}' | sudo tee ~/.docker/config.json
+        sudo service docker restart
+        docker version -f '{{.Client.Experimental}}'
+        docker version -f '{{.Server.Experimental}}'
+        docker buildx version
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@master
+      with:
+        platforms: all
+    - name: Login to DockerHub
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Build and push
+      if: github.event_name != 'pull_request'
+      run: make manifest
+    - name: Build and push latest
+      if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+      run: make manifest-latest

.github/workflows/release.yaml

@@ -0,0 +1,21 @@
+on:
+  release:
+    types: [created]
+name: Handle Release
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.5
+    - name: Make Directory with kgctl Binaries to Be Released
+      run: make release
+    - name: Publish Release
+      uses: skx/github-action-publish-binaries@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        args: 'bin/release/kgctl-*'

.gitignore

@@ -3,3 +3,5 @@
 .manifest*
 .push*
 bin/
+tmp/
+e2e/kind.yaml*

.travis.yml

@@ -1,31 +0,0 @@
-sudo: required
-
-dist: xenial
-
-language: go
-
-services:
-  - docker
-
-go:
-  - 1.14.2
-
-env:
-  - GO111MODULE=on DOCKER_CLI_EXPERIMENTAL=enabled
-
-before_install:
-  - sudo apt-get update && sudo apt-get -y install jq
-
-install: true
-
-script:
-  - make
-  - make clean
-  - make unit
-  - make lint
-  - make container
-
-after_success:
-  - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-  - make manifest && make manifest-latest

Dockerfile

@@ -1,15 +1,17 @@
 ARG FROM=alpine
-FROM alpine AS cni
+FROM $FROM AS cni
-ARG GOARCH
+ARG GOARCH=amd64
+ARG CNI_PLUGINS_VERSION=v0.9.1
 RUN apk add --no-cache curl && \
-    curl -Lo cni.tar.gz https://github.com/containernetworking/plugins/releases/download/v0.7.5/cni-plugins-$GOARCH-v0.7.5.tgz && \
+    curl -Lo cni.tar.gz https://github.com/containernetworking/plugins/releases/download/$CNI_PLUGINS_VERSION/cni-plugins-linux-$GOARCH-$CNI_PLUGINS_VERSION.tgz && \
     tar -xf cni.tar.gz
 
 FROM $FROM
 ARG GOARCH
+ARG ALPINE_VERSION=v3.12
 LABEL maintainer="squat <lserven@gmail.com>"
-RUN echo -e "https://alpine.global.ssl.fastly.net/alpine/v3.12/main\nhttps://alpine.global.ssl.fastly.net/alpine/v3.12/community" > /etc/apk/repositories && \
+RUN echo -e "https://alpine.global.ssl.fastly.net/alpine/$ALPINE_VERSION/main\nhttps://alpine.global.ssl.fastly.net/alpine/$ALPINE_VERSION/community" > /etc/apk/repositories && \
     apk add --no-cache ipset iptables ip6tables wireguard-tools
 COPY --from=cni bridge host-local loopback portmap /opt/cni/bin/
-COPY bin/$GOARCH/kg /opt/bin/
+COPY bin/linux/$GOARCH/kg /opt/bin/
 ENTRYPOINT ["/opt/bin/kg"]

MAINTAINERS.md

@@ -0,0 +1,6 @@
+# Core Maintainers of This Repository
+
+| Name               | GitHub                                         |
+|--------------------|------------------------------------------------|
+| Lucas Servén Marín | [@squat](https://github.com/squat)             |
+| Leon Löchner       | [@leonnicolas](https://github.com/leonnicolas) |

Makefile

@@ -1,14 +1,21 @@
 export GO111MODULE=on
-.PHONY: push container clean container-name container-latest push-latest fmt lint test unit vendor header generate client deepcopy informer lister openapi manifest manfest-latest manifest-annotate manifest manfest-latest manifest-annotate
+.PHONY: push container clean container-name container-latest push-latest fmt lint test unit vendor header generate crd client deepcopy informer lister manifest manfest-latest manifest-annotate manifest manfest-latest manifest-annotate release gen-docs e2e
 
-ARCH ?= amd64
+OS ?= $(shell go env GOOS)
+ARCH ?= $(shell go env GOARCH)
 ALL_ARCH := amd64 arm arm64
 DOCKER_ARCH := "amd64" "arm v7" "arm64 v8"
-BINS := $(addprefix bin/$(ARCH)/,kg kgctl)
+ifeq ($(OS),linux)
+    BINS := bin/$(OS)/$(ARCH)/kg bin/$(OS)/$(ARCH)/kgctl
+else
+    BINS := bin/$(OS)/$(ARCH)/kgctl
+endif
+RELEASE_BINS := $(addprefix bin/release/kgctl-, $(addprefix linux-, $(ALL_ARCH)) darwin-amd64 darwin-arm64 windows-amd64)
 PROJECT := kilo
 PKG := github.com/squat/$(PROJECT)
 REGISTRY ?= index.docker.io
 IMAGE ?= squat/$(PROJECT)
+FULLY_QUALIFIED_IMAGE := $(REGISTRY)/$(IMAGE)
 
 TAG := $(shell git describe --abbrev=0 --tags HEAD 2>/dev/null)
 COMMIT := $(shell git rev-parse HEAD)
@@ -25,20 +32,26 @@ SRC := $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 GO_FILES ?= $$(find . -name '*.go' -not -path './vendor/*')
 GO_PKGS ?= $$(go list ./... | grep -v "$(PKG)/vendor")
 
+CONTROLLER_GEN_BINARY := bin/controller-gen
 CLIENT_GEN_BINARY := bin/client-gen
+DOCS_GEN_BINARY := bin/docs-gen
 DEEPCOPY_GEN_BINARY := bin/deepcopy-gen
 INFORMER_GEN_BINARY := bin/informer-gen
 LISTER_GEN_BINARY := bin/lister-gen
-OPENAPI_GEN_BINARY := bin/openapi-gen
 GOLINT_BINARY := bin/golint
+EMBEDMD_BINARY := bin/embedmd
+KIND_BINARY := $(shell pwd)/bin/kind
+KUBECTL_BINARY := $(shell pwd)/bin/kubectl
+BASH_UNIT := $(shell pwd)/bin/bash_unit
+BASH_UNIT_FLAGS :=
 
-BUILD_IMAGE ?= golang:1.14.2-alpine
+BUILD_IMAGE ?= golang:1.16.5-alpine
-BASE_IMAGE ?= alpine:3.12
+BASE_IMAGE ?= alpine:3.13
 
 build: $(BINS)
 
 build-%:
-	@$(MAKE) --no-print-directory ARCH=$* build
+	@$(MAKE) --no-print-directory OS=$(word 1,$(subst -, ,$*)) ARCH=$(word 2,$(subst -, ,$*)) build
 
 container-latest-%:
 	@$(MAKE) --no-print-directory ARCH=$* container-latest
@@ -52,7 +65,7 @@ push-latest-%:
 push-%:
 	@$(MAKE) --no-print-directory ARCH=$* push
 
-all-build: $(addprefix build-, $(ALL_ARCH))
+all-build: $(addprefix build-$(OS)-, $(ALL_ARCH))
 
 all-container: $(addprefix container-, $(ALL_ARCH))
 
@@ -62,7 +75,13 @@ all-container-latest: $(addprefix container-latest-, $(ALL_ARCH))
 
 all-push-latest: $(addprefix push-latest-, $(ALL_ARCH))
 
-generate: client deepcopy informer lister openapi
+generate: client deepcopy informer lister crd
+
+crd: manifests/crds.yaml
+manifests/crds.yaml: pkg/k8s/apis/kilo/v1alpha1/types.go $(CONTROLLER_GEN_BINARY)
+	$(CONTROLLER_GEN_BINARY) crd \
+	paths=./pkg/k8s/apis/kilo/... \
+	output:crd:stdout | tail -n +3 > $@
 
 client: pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go
 pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go: .header pkg/k8s/apis/kilo/v1alpha1/types.go $(CLIENT_GEN_BINARY)
@@ -120,19 +139,12 @@ pkg/k8s/listers/kilo/v1alpha1/peer.go: .header pkg/k8s/apis/kilo/v1alpha1/types.
 	rm -r github.com || true
 	go fmt ./pkg/k8s/listers/...
 
-openapi: pkg/k8s/apis/kilo/v1alpha1/openapi_generated.go
+gen-docs: generate docs/api.md
-pkg/k8s/apis/kilo/v1alpha1/openapi_generated.go: pkg/k8s/apis/kilo/v1alpha1/types.go $(OPENAPI_GEN_BINARY)
+docs/api.md: pkg/k8s/apis/kilo/v1alpha1/types.go $(DOCS_GEN_BINARY)
-	$(OPENAPI_GEN_BINARY) \
+	$(DOCS_GEN_BINARY) $< > $@
-	--input-dirs $(PKG)/$(@D),k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/core/v1 \
-	--output-base $(CURDIR) \
-	--output-package ./$(@D) \
-	--logtostderr \
-	--report-filename /dev/null \
-	--go-header-file=.header
-	go fmt $@
 
 $(BINS): $(SRC) go.mod
-	@mkdir -p bin/$(ARCH)
+	@mkdir -p bin/$(word 2,$(subst /, ,$@))/$(word 3,$(subst /, ,$@))
 	@echo "building: $@"
 	@docker run --rm \
 	    -u $$(id -u):$$(id -g) \
@@ -140,8 +152,8 @@ $(BINS): $(SRC) go.mod
 	    -w /$(PROJECT) \
 	    $(BUILD_IMAGE) \
 	    /bin/sh -c " \
-	        GOARCH=$(ARCH) \
+	        GOARCH=$(word 3,$(subst /, ,$@)) \
-	        GOOS=linux \
+	        GOOS=$(word 2,$(subst /, ,$@)) \
 	        GOCACHE=/$(PROJECT)/.cache \
 		CGO_ENABLED=0 \
 		go build -mod=vendor -o $@ \
@@ -182,7 +194,22 @@ lint: header $(GOLINT_BINARY)
 unit:
 	go test -mod=vendor --race ./...
 
-test: lint unit
+test: lint unit e2e
+
+$(KIND_BINARY):
+	curl -Lo $@ https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-$(ARCH)
+	chmod +x $@
+
+$(KUBECTL_BINARY):
+	curl -Lo $@ https://dl.k8s.io/release/v1.21.0/bin/linux/$(ARCH)/kubectl
+	chmod +x $@
+
+$(BASH_UNIT):
+	curl -Lo $@ https://raw.githubusercontent.com/pgrange/bash_unit/v1.7.2/bash_unit
+	chmod +x $@
+
+e2e: container $(KIND_BINARY) $(KUBECTL_BINARY) $(BASH_UNIT) bin/$(OS)/$(ARCH)/kgctl
+	KILO_IMAGE=$(IMAGE):$(ARCH)-$(VERSION) KIND_BINARY=$(KIND_BINARY) KUBECTL_BINARY=$(KUBECTL_BINARY) KGCTL_BINARY=$(shell pwd)/bin/$(OS)/$(ARCH)/kgctl $(BASH_UNIT) $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/multi-cluster.sh ./e2e/teardown.sh
 
 header: .header
 	@HEADER=$$(cat .header); \
@@ -190,7 +217,7 @@ header: .header
 	FILES=; \
 	for f in $(GO_FILES); do \
 		for i in 0 1 2 3 4 5; do \
-			FILE=$$(tail -n +$$i $$f | head -n $$HEADER_LEN | sed "s/[0-9]\{4\}/YEAR/"); \
+			FILE=$$(t=$$(mktemp) && tail -n +$$i $$f > $$t && head -n $$HEADER_LEN $$t | sed "s/[0-9]\{4\}/YEAR/"); \
 			[ "$$FILE" = "$$HEADER" ] && continue 2; \
 		done; \
 		FILES="$$FILES$$f "; \
@@ -200,6 +227,13 @@ header: .header
 		exit 1; \
 	fi
 
+tmp/help.txt: bin/$(OS)/$(ARCH)/kg
+	mkdir -p tmp
+	bin//$(OS)/$(ARCH)/kg --help 2>&1 | head -n -1 > $@
+
+docs/kg.md: $(EMBEDMD_BINARY) tmp/help.txt
+	$(EMBEDMD_BINARY) -w $@
+
 website/docs/README.md: README.md
 	rm -rf website/static/img/graphs
 	find docs  -type f -name '*.md' | xargs -I{} sh -c 'cat $(@D)/$$(basename {} .md) > website/{}'
@@ -210,13 +244,15 @@ website/docs/README.md: README.md
 	sed -i 's/\.\/docs\///g' $@
 	find $(@D)  -type f -name '*.md' | xargs -I{} sed -i 's/\.\/\(.\+\.svg\)/\/img\/\1/g' {}
 	sed -i 's/graphs\//\/img\/graphs\//g' $@
+	# The next line is a workaround until mdx, docusaurus' markdown parser, can parse links with preceding brackets.
+	sed -i  's/\[\]\(\[.*\](.*)\)/\&#91;\&#93;\1/g' website/docs/api.md
 
-website/build/index.html: website/docs/README.md
+website/build/index.html: website/docs/README.md docs/api.md
 	yarn --cwd website install
 	yarn --cwd website build
 
 container: .container-$(ARCH)-$(VERSION) container-name
-.container-$(ARCH)-$(VERSION): $(BINS) Dockerfile
+.container-$(ARCH)-$(VERSION): bin/linux/$(ARCH)/kg Dockerfile
 	@i=0; for a in $(ALL_ARCH); do [ "$$a" = $(ARCH) ] && break; i=$$((i+1)); done; \
 	ia=""; iv=""; \
 	j=0; for a in $(DOCKER_ARCH); do \
@@ -227,7 +263,7 @@ container: .container-$(ARCH)-$(VERSION) container-name
 	@docker images -q $(IMAGE):$(ARCH)-$(VERSION) > $@
 
 container-latest: .container-$(ARCH)-$(VERSION)
-	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(IMAGE):$(ARCH)-latest
+	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(FULLY_QUALIFIED_IMAGE):$(ARCH)-latest
 	@echo "container: $(IMAGE):$(ARCH)-latest"
 
 container-name:
@@ -235,14 +271,15 @@ container-name:
 
 manifest: .manifest-$(VERSION) manifest-name
 .manifest-$(VERSION): Dockerfile $(addprefix push-, $(ALL_ARCH))
-	@docker manifest create --amend $(IMAGE):$(VERSION) $(addsuffix -$(VERSION), $(addprefix squat/$(PROJECT):, $(ALL_ARCH)))
+	@docker manifest create --amend $(FULLY_QUALIFIED_IMAGE):$(VERSION) $(addsuffix -$(VERSION), $(addprefix $(FULLY_QUALIFIED_IMAGE):, $(ALL_ARCH)))
 	@$(MAKE) --no-print-directory manifest-annotate-$(VERSION)
-	@docker manifest push $(IMAGE):$(VERSION) > $@
+	@docker manifest push $(FULLY_QUALIFIED_IMAGE):$(VERSION) > $@
 
 manifest-latest: Dockerfile $(addprefix push-latest-, $(ALL_ARCH))
-	@docker manifest create --amend $(IMAGE):latest $(addsuffix -latest, $(addprefix squat/$(PROJECT):, $(ALL_ARCH)))
+	@docker manifest rm $(FULLY_QUALIFIED_IMAGE):latest || echo no old manifest
+	@docker manifest create --amend $(FULLY_QUALIFIED_IMAGE):latest $(addsuffix -latest, $(addprefix $(FULLY_QUALIFIED_IMAGE):, $(ALL_ARCH)))
 	@$(MAKE) --no-print-directory manifest-annotate-latest
-	@docker manifest push $(IMAGE):latest
+	@docker manifest push $(FULLY_QUALIFIED_IMAGE):latest
 	@echo "manifest: $(IMAGE):latest"
 
 manifest-annotate: manifest-annotate-$(VERSION)
@@ -253,7 +290,7 @@ manifest-annotate-%:
 	    annotate=; \
 	    j=0; for da in $(DOCKER_ARCH); do \
 		if [ "$$j" -eq "$$i" ] && [ -n "$$da" ]; then \
-		    annotate="docker manifest annotate $(IMAGE):$* $(IMAGE):$$a-$* --os linux --arch"; \
+		    annotate="docker manifest annotate $(FULLY_QUALIFIED_IMAGE):$* $(FULLY_QUALIFIED_IMAGE):$$a-$* --os linux --arch"; \
 		    k=0; for ea in $$da; do \
 			[ "$$k" = 0 ] && annotate="$$annotate $$ea"; \
 			[ "$$k" != 0 ] && annotate="$$annotate --variant $$ea"; \
@@ -267,20 +304,29 @@ manifest-annotate-%:
 	done
 
 manifest-name:
-	@echo "manifest: $(IMAGE_ROOT):$(VERSION)"
+	@echo "manifest: $(IMAGE):$(VERSION)"
 
 push: .push-$(ARCH)-$(VERSION) push-name
 .push-$(ARCH)-$(VERSION): .container-$(ARCH)-$(VERSION)
-	@docker push $(REGISTRY)/$(IMAGE):$(ARCH)-$(VERSION)
+ifneq ($(REGISTRY),index.docker.io)
+	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(FULLY_QUALIFIED_IMAGE):$(ARCH)-$(VERSION)
+endif
+	@docker push $(FULLY_QUALIFIED_IMAGE):$(ARCH)-$(VERSION)
 	@docker images -q $(IMAGE):$(ARCH)-$(VERSION) > $@
 
 push-latest: container-latest
-	@docker push $(REGISTRY)/$(IMAGE):$(ARCH)-latest
+	@docker push $(FULLY_QUALIFIED_IMAGE):$(ARCH)-latest
 	@echo "pushed: $(IMAGE):$(ARCH)-latest"
 
 push-name:
 	@echo "pushed: $(IMAGE):$(ARCH)-$(VERSION)"
 
+release: $(RELEASE_BINS)
+$(RELEASE_BINS):
+	@make OS=$(word 2,$(subst -, ,$(@F))) ARCH=$(word 3,$(subst -, ,$(@F)))
+	mkdir -p $(@D)
+	cp bin/$(word 2,$(subst -, ,$(@F)))/$(word 3,$(subst -, ,$(@F)))/kgctl $@
+
 clean: container-clean bin-clean
 	rm -rf .cache
 
@@ -294,6 +340,9 @@ vendor:
 	go mod tidy
 	go mod vendor
 
+$(CONTROLLER_GEN_BINARY):
+	go build -mod=vendor -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen
+
 $(CLIENT_GEN_BINARY):
 	go build -mod=vendor -o $@ k8s.io/code-generator/cmd/client-gen
 
@@ -306,8 +355,11 @@ $(INFORMER_GEN_BINARY):
 $(LISTER_GEN_BINARY):
 	go build -mod=vendor -o $@ k8s.io/code-generator/cmd/lister-gen
 
-$(OPENAPI_GEN_BINARY):
+$(DOCS_GEN_BINARY): cmd/docs-gen/main.go
-	go build -mod=vendor -o $@ k8s.io/kube-openapi/cmd/openapi-gen
+	go build -mod=vendor -o $@ ./cmd/docs-gen
 
 $(GOLINT_BINARY):
 	go build -mod=vendor -o $@ golang.org/x/lint/golint
+
+$(EMBEDMD_BINARY):
+	go build -mod=vendor -o $@ github.com/campoy/embedmd

README.md

@@ -4,17 +4,22 @@
 
 Kilo is a multi-cloud network overlay built on WireGuard and designed for Kubernetes.
 
-[![Build Status](https://travis-ci.org/squat/kilo.svg?branch=master)](https://travis-ci.org/squat/kilo)
+[![Build Status](https://github.com/squat/kilo/workflows/CI/badge.svg)](https://github.com/squat/kilo/actions?query=workflow%3ACI)
 [![Go Report Card](https://goreportcard.com/badge/github.com/squat/kilo)](https://goreportcard.com/report/github.com/squat/kilo)
+[![Docker Pulls](https://img.shields.io/docker/pulls/squat/kilo)](https://hub.docker.com/r/squat/kilo)
+[![Slack](https://img.shields.io/badge/join%20slack-%23kilo-brightgreen.svg)](https://slack.k8s.io/)
 
 ## Overview
 
 Kilo connects nodes in a cluster by providing an encrypted layer 3 network that can span across data centers and public clouds.
+The Pod network created by Kilo is always fully connected, even when the nodes are in different networks or behind NAT.
 By allowing pools of nodes in different locations to communicate securely, Kilo enables the operation of multi-cloud clusters.
 Kilo's design allows clients to VPN to a cluster in order to securely access services running on the cluster.
 In addition to creating multi-cloud clusters, Kilo enables the creation of multi-cluster services, i.e. services that span across different Kubernetes clusters.
 
-## How it works
+An introductory video about Kilo from KubeCon EU 2019 can be found on [youtube](https://www.youtube.com/watch?v=iPz_DAOOCKA).
+
+## How It Works
 
 Kilo uses [WireGuard](https://www.wireguard.com/), a performant and secure VPN, to create a mesh between the different nodes in a cluster.
 The Kilo agent, `kg`, runs on every node in the cluster, setting up the public and private keys for the VPN as well as the necessary rules to route packets between locations.
@@ -26,15 +31,14 @@ This means that if a cluster uses, for example, Flannel for networking, Kilo can
 
 Kilo can be installed on any Kubernetes cluster either pre- or post-bring-up.
 
-### Step 1: install WireGuard
+### Step 1: get WireGuard
 
-Kilo requires the WireGuard kernel module on all nodes in the cluster.
+Kilo requires the WireGuard kernel module to be loaded on all nodes in the cluster.
-For most Linux distributions, this can be installed using the system package manager.
+Starting at Linux 5.6, the kernel includes WireGuard in-tree; Linux distributions with older kernels will need to install WireGuard.
-For Container Linux, WireGuard can be easily installed using a DaemonSet:
+For most Linux distributions, this can be done using the system package manager.
+[See the WireGuard website for up-to-date instructions for installing WireGuard](https://www.wireguard.com/install/).
 
-```shell
+Clusters with nodes on which the WireGuard kernel module cannot be installed can use Kilo by leveraging a [userspace WireGuard implementation](./docs/userspace-wireguard.md).
-kubectl apply -f https://raw.githubusercontent.com/squat/modulus/master/wireguard/daemonset.yaml
-```
 
 ### Step 2: open WireGuard port
 
@@ -68,25 +72,29 @@ Kilo can be installed by deploying a DaemonSet to the cluster.
 To run Kilo on kubeadm:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-kubeadm.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-kubeadm.yaml
 ```
 
 To run Kilo on bootkube:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-bootkube.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-bootkube.yaml
 ```
 
 To run Kilo on Typhoon:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-typhoon.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-typhoon.yaml
 ```
 
 To run Kilo on k3s:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-k3s.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s.yaml
 ```
 
 ## Add-on Mode
@@ -98,15 +106,16 @@ Kilo currently supports running on top of Flannel.
 For example, to run Kilo on a Typhoon cluster running Flannel:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-typhoon-flannel.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-typhoon-flannel.yaml
 ```
 
-[See the manifests directory for more examples](https://github.com/squat/kilo/tree/master/manifests).
+[See the manifests directory for more examples](https://github.com/squat/kilo/tree/main/manifests).
 
 ## VPN
 
 Kilo also enables peers outside of a Kubernetes cluster to connect to the VPN, allowing cluster applications to securely access external services and permitting developers and support to securely debug cluster resources.
-In order to declare a peer, start by defining a Kilo peer resource:
+In order to declare a peer, start by defining a Kilo Peer resource:
 
 ```shell
 cat <<'EOF' | kubectl apply -f -
@@ -147,7 +156,7 @@ for n in $(kubectl --kubeconfig $KUBECONFIG2 get no -o name | cut -d'/' -f2); do
     kgctl --kubeconfig $KUBECONFIG2 showconf node $n --as-peer -o yaml --allowed-ips $SERVICECIDR2 | kubectl --kubeconfig $KUBECONFIG1 apply -f -
 done
 # Create a Service in cluster2 to mirror the Service in cluster1.
-cat <<'EOF' | kubectl --kubeconfig $KUBECONFIG2 apply -f -
+cat <<EOF | kubectl --kubeconfig $KUBECONFIG2 apply -f -
 apiVersion: v1
 kind: Service
 metadata:

cmd/docs-gen/main.go

@@ -0,0 +1,292 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file was adapted from https://github.com/prometheus-operator/prometheus-operator/blob/master/cmd/po-docgen/api.go.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"os"
+	"reflect"
+	"strings"
+)
+
+const (
+	firstParagraph = `# API
+This document is a reference of the API types introduced by Kilo.
+
+> **Note**: this document is generated from code comments. When contributing a change to this document, please do so by changing the code comments.`
+)
+
+var (
+	links = map[string]string{
+		"metav1.ObjectMeta": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#objectmeta-v1-meta",
+		"metav1.ListMeta":   "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#listmeta-v1-meta",
+	}
+
+	selfLinks = map[string]string{}
+	typesDoc  = map[string]KubeTypes{}
+)
+
+func toSectionLink(name string) string {
+	name = strings.ToLower(name)
+	name = strings.Replace(name, " ", "-", -1)
+	return name
+}
+
+func printTOC(types []KubeTypes) {
+	fmt.Printf("\n## Table of Contents\n")
+	for _, t := range types {
+		strukt := t[0]
+		if len(t) > 1 {
+			fmt.Printf("* [%s](#%s)\n", strukt.Name, toSectionLink(strukt.Name))
+		}
+	}
+}
+
+func printAPIDocs(paths []string) {
+	fmt.Println(firstParagraph)
+
+	types := parseDocumentationFrom(paths)
+	for _, t := range types {
+		strukt := t[0]
+		selfLinks[strukt.Name] = "#" + strings.ToLower(strukt.Name)
+		typesDoc[toLink(strukt.Name)] = t[1:]
+	}
+
+	// we need to parse once more to now add the self links and the inlined fields
+	types = parseDocumentationFrom(paths)
+
+	printTOC(types)
+
+	for _, t := range types {
+		strukt := t[0]
+		if len(t) > 1 {
+			fmt.Printf("\n## %s\n\n%s\n\n", strukt.Name, strukt.Doc)
+
+			fmt.Println("| Field | Description | Scheme | Required |")
+			fmt.Println("| ----- | ----------- | ------ | -------- |")
+			fields := t[1:]
+			for _, f := range fields {
+				fmt.Println("|", f.Name, "|", f.Doc, "|", f.Type, "|", f.Mandatory, "|")
+			}
+			fmt.Println("")
+			fmt.Println("[Back to TOC](#table-of-contents)")
+		}
+	}
+}
+
+// Pair of strings. We need the name of fields and the doc.
+type Pair struct {
+	Name, Doc, Type string
+	Mandatory       bool
+}
+
+// KubeTypes is an array to represent all available types in a parsed file. [0] is for the type itself
+type KubeTypes []Pair
+
+// parseDocumentationFrom gets all types' documentation and returns them as an
+// array. Each type is again represented as an array (we have to use arrays as we
+// need to be sure of the order of the fields). This function returns fields and
+// struct definitions that have no documentation as {name, ""}.
+func parseDocumentationFrom(srcs []string) []KubeTypes {
+	var docForTypes []KubeTypes
+
+	for _, src := range srcs {
+		pkg := astFrom(src)
+
+		for _, kubType := range pkg.Types {
+			if structType, ok := kubType.Decl.Specs[0].(*ast.TypeSpec).Type.(*ast.StructType); ok {
+				var ks KubeTypes
+				ks = append(ks, Pair{kubType.Name, fmtRawDoc(kubType.Doc), "", false})
+
+				for _, field := range structType.Fields.List {
+					// Skip fields that are not tagged.
+					if field.Tag == nil {
+						os.Stderr.WriteString(fmt.Sprintf("Tag is nil, skipping field: %v of type %v\n", field, field.Type))
+						continue
+					}
+					// Treat inlined fields separately as we don't want the original types to appear in the doc.
+					if isInlined(field) {
+						// Skip external types, as we don't want their content to be part of the API documentation.
+						if isInternalType(field.Type) {
+							ks = append(ks, typesDoc[fieldType(field.Type)]...)
+						}
+						continue
+					}
+
+					typeString := fieldType(field.Type)
+					fieldMandatory := fieldRequired(field)
+					if n := fieldName(field); n != "-" {
+						fieldDoc := fmtRawDoc(field.Doc.Text())
+						ks = append(ks, Pair{n, fieldDoc, typeString, fieldMandatory})
+					}
+				}
+				docForTypes = append(docForTypes, ks)
+			}
+		}
+	}
+
+	return docForTypes
+}
+
+func astFrom(filePath string) *doc.Package {
+	fset := token.NewFileSet()
+	m := make(map[string]*ast.File)
+
+	f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments)
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+
+	m[filePath] = f
+	apkg, _ := ast.NewPackage(fset, m, nil, nil)
+
+	return doc.New(apkg, "", 0)
+}
+
+func fmtRawDoc(rawDoc string) string {
+	var buffer bytes.Buffer
+	delPrevChar := func() {
+		if buffer.Len() > 0 {
+			buffer.Truncate(buffer.Len() - 1) // Delete the last " " or "\n"
+		}
+	}
+
+	// Ignore all lines after ---
+	rawDoc = strings.Split(rawDoc, "---")[0]
+
+	for _, line := range strings.Split(rawDoc, "\n") {
+		line = strings.TrimRight(line, " ")
+		leading := strings.TrimLeft(line, " ")
+		switch {
+		case len(line) == 0: // Keep paragraphs
+			delPrevChar()
+			buffer.WriteString("\n\n")
+		case strings.HasPrefix(leading, "TODO"): // Ignore one line TODOs
+		case strings.HasPrefix(leading, "+"): // Ignore instructions to go2idl
+		default:
+			if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
+				delPrevChar()
+				line = "\n" + line + "\n" // Replace it with newline. This is useful when we have a line with: "Example:\n\tJSON-someting..."
+			} else {
+				line += " "
+			}
+			buffer.WriteString(line)
+		}
+	}
+
+	postDoc := strings.TrimRight(buffer.String(), "\n")
+	postDoc = strings.Replace(postDoc, "\\\"", "\"", -1) // replace user's \" to "
+	postDoc = strings.Replace(postDoc, "\"", "\\\"", -1) // Escape "
+	postDoc = strings.Replace(postDoc, "\n", "\\n", -1)
+	postDoc = strings.Replace(postDoc, "\t", "\\t", -1)
+	postDoc = strings.Replace(postDoc, "|", "\\|", -1)
+
+	return postDoc
+}
+
+func toLink(typeName string) string {
+	selfLink, hasSelfLink := selfLinks[typeName]
+	if hasSelfLink {
+		return wrapInLink(typeName, selfLink)
+	}
+
+	link, hasLink := links[typeName]
+	if hasLink {
+		return wrapInLink(typeName, link)
+	}
+
+	return typeName
+}
+
+func wrapInLink(text, link string) string {
+	return fmt.Sprintf("[%s](%s)", text, link)
+}
+
+func isInlined(field *ast.Field) bool {
+	jsonTag := reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+	return strings.Contains(jsonTag, "inline")
+}
+
+func isInternalType(typ ast.Expr) bool {
+	switch typ := typ.(type) {
+	case *ast.SelectorExpr:
+		pkg := typ.X.(*ast.Ident)
+		return strings.HasPrefix(pkg.Name, "monitoring")
+	case *ast.StarExpr:
+		return isInternalType(typ.X)
+	case *ast.ArrayType:
+		return isInternalType(typ.Elt)
+	case *ast.MapType:
+		return isInternalType(typ.Key) && isInternalType(typ.Value)
+	default:
+		return true
+	}
+}
+
+// fieldName returns the name of the field as it should appear in JSON format
+// "-" indicates that this field is not part of the JSON representation
+func fieldName(field *ast.Field) string {
+	jsonTag := reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+	jsonTag = strings.Split(jsonTag, ",")[0]                                              // This can return "-"
+	if jsonTag == "" {
+		if field.Names != nil {
+			return field.Names[0].Name
+		}
+		return field.Type.(*ast.Ident).Name
+	}
+	return jsonTag
+}
+
+// fieldRequired returns whether a field is a required field.
+func fieldRequired(field *ast.Field) bool {
+	jsonTag := ""
+	if field.Tag != nil {
+		jsonTag = reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+		return !strings.Contains(jsonTag, "omitempty")
+	}
+
+	return false
+}
+
+func fieldType(typ ast.Expr) string {
+	switch typ := typ.(type) {
+	case *ast.Ident:
+		return toLink(typ.Name)
+	case *ast.StarExpr:
+		return "*" + toLink(fieldType(typ.X))
+	case *ast.SelectorExpr:
+		pkg := typ.X.(*ast.Ident)
+		t := typ.Sel
+		return toLink(pkg.Name + "." + t.Name)
+	case *ast.ArrayType:
+		return "[]" + toLink(fieldType(typ.Elt))
+	case *ast.MapType:
+		return "map[" + toLink(fieldType(typ.Key)) + "]" + toLink(fieldType(typ.Value))
+	default:
+		return ""
+	}
+}
+
+func main() {
+	printAPIDocs(os.Args[1:])
+}

cmd/kg/main.go

@@ -16,7 +16,6 @@ package main
 
 import (
 	"errors"
-	"flag"
 	"fmt"
 	"net"
 	"net/http"
@@ -24,12 +23,14 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	flag "github.com/spf13/pflag"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -80,6 +81,7 @@ var (
 func Main() error {
 	backend := flag.String("backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
 	cleanUpIface := flag.Bool("clean-up-interface", false, "Should Kilo delete its interface when it shuts down?")
+	createIface := flag.Bool("create-interface", true, "Should kilo create an interface on startup?")
 	cni := flag.Bool("cni", true, "Should Kilo manage the node's CNI configuration?")
 	cniPath := flag.String("cni-path", mesh.DefaultCNIPath, "Path to CNI config.")
 	compatibility := flag.String("compatibility", "", fmt.Sprintf("Should Kilo run in compatibility mode? Possible values: %s", availableCompatibilities))
@@ -92,9 +94,11 @@ func Main() error {
 	local := flag.Bool("local", true, "Should Kilo manage routes within a location?")
 	logLevel := flag.String("log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels))
 	master := flag.String("master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
+	topologyLabel := flag.String("topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
 	var port uint
 	flag.UintVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
 	subnet := flag.String("subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.")
+	resyncPeriod := flag.Duration("resync-period", 30*time.Second, "How often should the Kilo controllers reconcile?")
 	printVersion := flag.Bool("version", false, "Print version and exit")
 	flag.Parse()
 
@@ -171,12 +175,12 @@ func Main() error {
 		c := kubernetes.NewForConfigOrDie(config)
 		kc := kiloclient.NewForConfigOrDie(config)
 		ec := apiextensions.NewForConfigOrDie(config)
-		b = k8s.New(c, kc, ec)
+		b = k8s.New(c, kc, ec, *topologyLabel)
 	default:
 		return fmt.Errorf("backend %v unknown; possible values are: %s", *backend, availableBackends)
 	}
 
-	m, err := mesh.New(b, enc, gr, *hostname, uint32(port), s, *local, *cni, *cniPath, *iface, *cleanUpIface, log.With(logger, "component", "kilo"))
+	m, err := mesh.New(b, enc, gr, *hostname, uint32(port), s, *local, *cni, *cniPath, *iface, *cleanUpIface, *createIface, *resyncPeriod, log.With(logger, "component", "kilo"))
 	if err != nil {
 		return fmt.Errorf("failed to create Kilo mesh: %v", err)
 	}

cmd/kgctl/graph.go

@@ -38,6 +38,11 @@ func runGraph(_ *cobra.Command, _ []string) error {
 	if err != nil {
 		return fmt.Errorf("failed to list peers: %v", err)
 	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = optainGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to obtain granularity: %w", err)
+	}
+
 	var hostname string
 	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
@@ -60,7 +65,7 @@ func runGraph(_ *cobra.Command, _ []string) error {
 			peers[p.Name] = p
 		}
 	}
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, []byte{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}

cmd/kgctl/main.go

@@ -15,8 +15,10 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -46,6 +48,7 @@ var (
 	availableGranularities = strings.Join([]string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.AutoGranularity),
 	}, ", ")
 	availableLogLevels = strings.Join([]string{
 		logLevelAll,
@@ -63,6 +66,7 @@ var (
 	backend       string
 	granularity   string
 	kubeconfig    string
+	topologyLabel string
 )
 
 func runRoot(_ *cobra.Command, _ []string) error {
@@ -70,6 +74,7 @@ func runRoot(_ *cobra.Command, _ []string) error {
 	switch opts.granularity {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.AutoGranularity:
 	default:
 		return fmt.Errorf("mesh granularity %v unknown; posible values are: %s", granularity, availableGranularities)
 	}
@@ -83,7 +88,7 @@ func runRoot(_ *cobra.Command, _ []string) error {
 		c := kubernetes.NewForConfigOrDie(config)
 		kc := kiloclient.NewForConfigOrDie(config)
 		ec := apiextensions.NewForConfigOrDie(config)
-		opts.backend = k8s.New(c, kc, ec)
+		opts.backend = k8s.New(c, kc, ec, topologyLabel)
 	default:
 		return fmt.Errorf("backend %v unknown; posible values are: %s", backend, availableBackends)
 	}
@@ -107,9 +112,14 @@ func main() {
 		Version:           version.Version,
 	}
 	cmd.PersistentFlags().StringVar(&backend, "backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
-	cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
+	cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.AutoGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
-	cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig.")
+	defaultKubeconfig := os.Getenv("KUBECONFIG")
+	if _, err := os.Stat(defaultKubeconfig); os.IsNotExist(err) {
+		defaultKubeconfig = filepath.Join(os.Getenv("HOME"), ".kube/config")
+	}
+	cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", defaultKubeconfig, "Path to kubeconfig.")
 	cmd.PersistentFlags().Uint32Var(&opts.port, "port", mesh.DefaultKiloPort, "The WireGuard port over which the nodes communicate.")
+	cmd.PersistentFlags().StringVar(&topologyLabel, "topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
 
 	for _, subCmd := range []*cobra.Command{
 		graph(),
@@ -123,3 +133,20 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func optainGranularity(gr mesh.Granularity, ns []*mesh.Node) (mesh.Granularity, error) {
+	if gr == mesh.AutoGranularity {
+		if len(ns) == 0 {
+			return gr, errors.New("could not get any nodes")
+		}
+		ret := mesh.Granularity(ns[0].Granularity)
+		switch ret {
+		case mesh.LogicalGranularity:
+		case mesh.FullGranularity:
+		default:
+			return ret, fmt.Errorf("mesh granularity %v is not supported", opts.granularity)
+		}
+		return ret, nil
+	}
+	return gr, nil
+}

cmd/kgctl/showconf.go

@@ -121,6 +121,10 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("failed to list peers: %v", err)
 	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = optainGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to obtain granularity: %w", err)
+	}
 	hostname := args[0]
 	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
@@ -147,7 +151,7 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		}
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, []byte{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}
@@ -208,6 +212,10 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("failed to list peers: %v", err)
 	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = optainGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to obtain granularity: %w", err)
+	}
 	var hostname string
 	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
@@ -236,7 +244,7 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		return fmt.Errorf("did not find any peer named %q in the cluster", peer)
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet, peers[peer].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet, peers[peer].PersistentKeepalive, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}
@@ -289,9 +297,16 @@ func translatePeer(peer *wireguard.Peer) *v1alpha1.Peer {
 		aips = append(aips, aip.String())
 	}
 	var endpoint *v1alpha1.PeerEndpoint
-	if peer.Endpoint != nil && peer.Endpoint.Port > 0 && peer.Endpoint.IP != nil {
+	if peer.Endpoint != nil && peer.Endpoint.Port > 0 && (peer.Endpoint.IP != nil || peer.Endpoint.DNS != "") {
+		var ip string
+		if peer.Endpoint.IP != nil {
+			ip = peer.Endpoint.IP.String()
+		}
 		endpoint = &v1alpha1.PeerEndpoint{
-			IP:   peer.Endpoint.IP.String(),
+			DNSOrIP: v1alpha1.DNSOrIP{
+				DNS: peer.Endpoint.DNS,
+				IP:  ip,
+			},
 			Port: peer.Endpoint.Port,
 		}
 	}

docs/annotations.md

@@ -5,10 +5,11 @@ The following annotations can be added to any Kubernetes Node object to configur
 |Name|type|examples|
 |----|----|-------|
 |[kilo.squat.ai/force-endpoint](#force-endpoint)|host:port|`55.55.55.55:51820`, `example.com:1337`|
-|[kilo.squat.ai/force-internal-ip](#force-internal-ip)|CIDR|`55.55.55.55/32`|
+|[kilo.squat.ai/force-internal-ip](#force-internal-ip)|CIDR|`55.55.55.55/32`, `"-"`,`""`|
 |[kilo.squat.ai/leader](#leader)|string|`""`, `true`|
 |[kilo.squat.ai/location](#location)|string|`gcp-east`, `lab`|
 |[kilo.squat.ai/persistent-keepalive](#persistent-keepalive)|uint|`10`|
+|[kilo.squat.ai/allowed-location-ips](#allowed-location-ips)|CIDR|`66.66.66.66/32`|
 
 ### force-endpoint
 In order to create links between locations, Kilo requires at least one node in each location to have an endpoint, ie a `host:port` combination, that is routable from the other locations.
@@ -25,6 +26,7 @@ Kilo routes packets destined for nodes inside the same logical location using th
 The Kilo agent running on each node will use heuristics to automatically detect a private IP address for the node; however, in some circumstances it may be necessary to explicitly configure the IP address, for example:
  * _multiple private IP addresses_: if a node has multiple private IPs but one is preferred, then the preferred IP address should be specified;
  * _IPv6_: if a node has both private IPv4 and IPv6 addresses and the Kilo network should operate over IPv6, then the IPv6 address should be specified.
+ * _disable private IP with "-" or ""_: a node has a private and public address, but the private address ought to be ignored.
 
 ### leader
 By default, Kilo creates a network mesh at the data-center granularity.
@@ -34,7 +36,7 @@ In some situations it may be desirable to manually select the leader for a locat
  * _firewall_: Kilo requires an open UDP port, which defaults to 51820, to communicate between locations; if only one node is configured to have that port open, then that node should be given the leader annotation;
  * _bandwidth_: if certain nodes in the cluster have a higher bandwidth or lower latency Internet connection, then those nodes should be given the leader annotation.
 
-_Note_: multiple nodes within a single location can be given the leader annotation; in this case, Kilo will select one leader from the set of annotated nodes. 
+> **Note**: multiple nodes within a single location can be given the leader annotation; in this case, Kilo will select one leader from the set of annotated nodes.
 
 ### location
 Kilo allows nodes in different logical or physical locations to route packets to one-another.
@@ -42,7 +44,7 @@ In order to know what connections to create, Kilo needs to know which nodes are
 Kilo will try to infer each node's location from the [topology.kubernetes.io/region](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#topologykubernetesioregion) node label.
 If the label is not present for a node, for example if running a bare-metal cluster or on an unsupported cloud provider, then the location annotation should be specified.
 
-_Note_: all nodes without a defined location will be considered to be in the default location `""`.
+> **Note**: all nodes without a defined location will be considered to be in the default location `""`.
 
 ### persistent-keepalive
 In certain deployments, cluster nodes may be located behind NAT or a firewall, e.g. edge nodes located behind a commodity router.
@@ -51,3 +53,10 @@ In order for a node behind NAT to receive packets from nodes outside of the NATe
 The frequency of emission of these keepalive packets can be controlled by setting the persistent-keepalive annotation on the node behind NAT.
 The annotated node will use the specified value will as the persistent-keepalive interval for all of its peers.
 For more background, [see the WireGuard documentation on NAT and firewall traversal](https://www.wireguard.com/quickstart/#nat-and-firewall-traversal-persistence).
+
+### allowed-location-ips
+It is possible to add allowed-location-ips to a location by annotating any node within that location.
+Adding allowed-location-ips to a location makes these IPs routable from other locations as well.
+
+In an example deployment of Kilo with two locations A and B, a printer in location A can be accessible from nodes and pods in location B.
+Additionally, Kilo Peers can use the printer in location A.

docs/api.md

@@ -0,0 +1,69 @@
+# API
+This document is a reference of the API types introduced by Kilo.
+
+> **Note**: this document is generated from code comments. When contributing a change to this document, please do so by changing the code comments.
+
+## Table of Contents
+* [DNSOrIP](#dnsorip)
+* [Peer](#peer)
+* [PeerEndpoint](#peerendpoint)
+* [PeerList](#peerlist)
+* [PeerSpec](#peerspec)
+
+## DNSOrIP
+
+DNSOrIP represents either a DNS name or an IP address. When both are given, the IP address, as it is more specific, override the DNS name.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| dns | DNS must be a valid RFC 1123 subdomain. | string | false |
+| ip | IP must be a valid IP address. | string | false |
+
+[Back to TOC](#table-of-contents)
+
+## Peer
+
+Peer is a WireGuard peer that should have access to the VPN.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| metadata | Standard object’s metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#metadata | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#objectmeta-v1-meta) | false |
+| spec | Specification of the desired behavior of the Kilo Peer. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status | [PeerSpec](#peerspec) | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerEndpoint
+
+PeerEndpoint represents a WireGuard endpoint, which is an IP:port tuple.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| dnsOrIP | DNSOrIP is a DNS name or an IP address. | [DNSOrIP](#dnsorip) | true |
+| port | Port must be a valid port number. | uint32 | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerList
+
+PeerList is a list of peers.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| metadata | Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | [metav1.ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#listmeta-v1-meta) | false |
+| items | List of peers. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md | [][Peer](#peer) | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerSpec
+
+PeerSpec is the description and configuration of a peer.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| allowedIPs | AllowedIPs is the list of IP addresses that are allowed for the given peer's tunnel. | []string | true |
+| endpoint | Endpoint is the initial endpoint for connections to the peer. | *[PeerEndpoint](#peerendpoint) | false |
+| persistentKeepalive | PersistentKeepalive is the interval in seconds of the emission of keepalive packets by the peer. This defaults to 0, which disables the feature. | int | false |
+| presharedKey | PresharedKey is the optional symmetric encryption key for the peer. | string | false |
+| publicKey | PublicKey is the WireGuard public key for the peer. | string | true |
+
+[Back to TOC](#table-of-contents)

docs/building_kilo.md

@@ -0,0 +1,100 @@
+# Build and Test Kilo
+
+This document describes how you can build and test Kilo.
+
+To follow along, you need to install the following utilities:
+ - `go` not for building but formatting the code and running unit tests
+ - `make`
+ - `jq`
+ - `git`
+ - `curl`
+ - `docker`
+
+## Getting Started
+
+Clone the Repository and `cd` into it.
+```shell
+git clone https://github.com/squat/kilo.git
+cd kilo
+```
+
+## Build
+
+For consistency, the Kilo binaries are compiled in a Docker container, so make sure the `docker` package is installed and the daemon is running.
+
+### Compile Binaries
+
+To compile the `kg` and `kgctl` binaries run:
+```shell
+make
+```
+Binaries are always placed in a directory corresponding to the local system's OS and architecture following the pattern `bin/<os>/<architecture>/`, so on an AMD64 machine running Linux, the binaries will be stored in `bin/linux/amd64/`.
+
+You can build the binaries for a different architecture by setting the `ARCH` environment variable before invoking `make`, e.g.:
+```shell
+ARCH=<arm|arm64|amd64> make
+```
+
+Likewise, to build `kg` for another OS, set the `OS` environment variable before invoking `make`:
+```shell
+OS=<windows|darwin|linux> make
+```
+## Test
+
+To execute the unit tests, run:
+```shell
+make unit
+```
+
+To lint the code in the repository, run:
+```shell
+make lint
+```
+
+To execute basic end to end tests, run:
+```shell
+make e2e
+```
+> **Note**: The end to end tests are currently flaky, so try running them again if they fail.
+
+To instead run all of the tests with a single command, run:
+```shell
+make test
+```
+
+## Build and Push the Container Images
+
+If you want to build containers for a processor architecture that is different from your computer's, then you will first need to configure QEMU as the interpreter for binaries built for non-native architectures:
+```shell
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+
+Set the `$IMAGE` environment variable to `<your Docker Hub user name>/kilo`.
+This way the generated container images and manifests will be named accordingly.
+By skipping this step, you will be able to tag images but will not be able to push the containers and manifests to your own Docker Hub.
+```shell
+export IMAGE=<docker hub user name>/kilo
+```
+
+If you want to use a different container registry, run:
+```shell
+export REGISTRY=<your registry without a trailing slash>
+```
+
+To build containers with the `kg` image for `arm`, `arm64` and `amd64`, run:
+```shell
+make all-container
+```
+
+Push the container images and build a manifest with:
+```shell
+make manifest
+```
+
+To tag and push the manifest with `latest`, run:
+```shell
+make manifest-latest
+```
+
+Now you can deploy the custom build of Kilo to your cluster.
+If you are already running Kilo, change the image from `squat/kilo` to `[registry/]<username>/kilo[:sha]`.

docs/building_website.md

@@ -0,0 +1,44 @@
+# Build and Test the Website
+
+You may have noticed that the `markdown` files in the `/docs` directory are also displayed on [Kilo's website](https://kilo.squat.ai/).
+If you want to add documentation to Kilo, you can start a local webserver to check out how the website would look like.
+
+## Requirements
+
+Install [yarn](https://yarnpkg.com/getting-started/install).
+
+## Build and Run
+
+The markdown files for the website are located in `/website/docs` and are generated from the like-named markdown files in the `/docs` directory and from the corresponding header files without the `.md` extension in the `/website/docs` directory.
+To generate the markdown files in `/website/docs`, run:
+```shell
+make website/docs/README.md
+```
+
+Next, build the website itself by installing the `node_modules` and building the website's HTML from the generated markdown:
+```shell
+make website/build/index.html
+```
+
+Now, start the website server with:
+```shell
+yarn --cwd website start
+```
+This command should have opened a browser window with the website; if not, open your browser and point it to `http://localhost:3000`.
+
+If you make changes to any of the markdown files in `/docs` and want to reload the local `node` server, run:
+```shell
+make website/docs/README.md -B
+```
+
+You can execute the above while the node server is running and the website will be rebuilt and reloaded automatically.
+
+## Add a New File to the Docs
+
+If you add a new file to the `/docs` directory, you also need to create a corresponding header file containing the front-matter in `/website/docs/`.
+Then, regenerate the markdown for the website with the command:
+```shell
+make website/docs/README.md
+```
+Edit `/website/sidebars.js` accordingly.
+> **Note**: The `id` in the header file `/website/docs/<new file>` must match the `id` specified in `website/sidebars.js`.

docs/kg.md

@@ -0,0 +1,40 @@
+# kg
+
+`kg` is the Kilo agent that runs on every Kubernetes node in a Kilo mesh.
+It performs several key functions, including:
+* adding the node to the Kilo mesh;
+* installing CNI configuration on the node;
+* configuring the WireGuard network interface; and
+* maintaining routing table entries and iptables rules.
+
+`kg` is typically installed on all nodes of a Kubernetes cluster using a DaemonSet.
+Example manifests can be found [in the manifests directory](https://github.com/squat/kilo/tree/main/manifests).
+
+## Usage
+
+The behavior of `kg` can be configured using the command line flags listed below.
+
+[embedmd]:# (../tmp/help.txt)
+```txt
+Usage of bin//linux/amd64/kg:
+      --backend string            The backend for the mesh. Possible values: kubernetes (default "kubernetes")
+      --clean-up-interface        Should Kilo delete its interface when it shuts down?
+      --cni                       Should Kilo manage the node's CNI configuration? (default true)
+      --cni-path string           Path to CNI config. (default "/etc/cni/net.d/10-kilo.conflist")
+      --compatibility string      Should Kilo run in compatibility mode? Possible values: flannel
+      --create-interface          Should kilo create an interface on startup? (default true)
+      --encapsulate string        When should Kilo encapsulate packets within a location? Possible values: never, crosssubnet, always (default "always")
+      --hostname string           Hostname of the node on which this process is running.
+      --interface string          Name of the Kilo interface to use; if it does not exist, it will be created. (default "kilo0")
+      --kubeconfig string         Path to kubeconfig.
+      --listen string             The address at which to listen for health and metrics. (default ":1107")
+      --local                     Should Kilo manage routes within a location? (default true)
+      --log-level string          Log level to use. Possible values: all, debug, info, warn, error, none (default "info")
+      --master string             The address of the Kubernetes API server (overrides any value in kubeconfig).
+      --mesh-granularity string   The granularity of the network mesh to create. Possible values: location, full (default "location")
+      --port uint                 The port over which WireGuard peers should communicate. (default 51820)
+      --resync-period duration    How often should the Kilo controllers reconcile? (default 30s)
+      --subnet string             CIDR from which to allocate addresses for WireGuard interfaces. (default "10.4.0.0/16")
+      --topology-label string     Kubernetes node label used to group nodes into logical locations. (default "topology.kubernetes.io/region")
+      --version                   Print version and exit
+```

docs/kgctl.md

@@ -6,14 +6,31 @@ This tool can be used to understand a mesh's topology, get the WireGuard configu
 
 ## Installation
 
-Installing `kgctl` currently requires building the binary from source.
+The `kgctl` binary is automatically compiled for Linux, macOS, and Windows for every release of Kilo and can be downloaded from [the GitHub releases page](https://github.com/squat/kilo/releases/latest).
-*Note*: the [Go toolchain must be installed](https://golang.org/doc/install) in order to build the binary.
+
-To build and install `kgctl`, simply run:
+### Building from Source
+Kilo is written in Golang and as a result the [Go toolchain must be installed](https://golang.org/doc/install) in order to build the `kgctl` binary.
+To download the Kilo source code and then build and install `kgctl` using the latest commit all with a single command, run:
 
 ```shell
-go install github.com/squat/kilo/cmd/kgctl
+go install github.com/squat/kilo/cmd/kgctl@latest
 ```
 
+Alternatively, `kgctl` can be built and installed based on specific version of the code by specifying a Git tag or hash, e.g.:
+
+```shell
+go install github.com/squat/kilo/cmd/kgctl@0.2.0
+```
+
+When working on Kilo locally, it can be helpful to build and test the `kgctl` binary as part of the development cycle.
+In order to build a binary from a local checkout of the Git repository, run:
+
+```shell
+make
+```
+
+This will produce a `kgctl` binary at `./bin/<your-os>/<your-architecture>/kgctl`.
+
 ## Commands
 
 |Command|Syntax|Description|

docs/multi-cluster-services.md

@@ -5,7 +5,7 @@ This enables clusters to provide services to other clusters over a secure connec
 For example, a cluster on AWS with access to GPUs could run a machine learning service that could be consumed by workloads running in a another location, e.g. an on-prem cluster without GPUs.
 Unlike services exposed via Ingresses or NodePort Services, multi-cluster services can remain private and internal to the clusters.
 
-*Note*: in order for connected clusters to be fully routable, the allowed IPs that they declare must be non-overlapping, i.e. the Kilo, pod, and service CIDRs.
+> **Note**: in order for connected clusters to be fully routable, the allowed IPs that they declare must be non-overlapping, i.e. the Kilo, pod, and service CIDRs.
 
 ## Getting Started
 

docs/network-policies.md

@@ -10,7 +10,7 @@ Support for [Kubernetes network policies](https://kubernetes.io/docs/concepts/se
 The following command adds network policy support by deploying kube-router to work alongside Kilo:
 
 ```shell
-kubectl apply -f kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kube-router.yaml
+kubectl apply -f kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kube-router.yaml
 ```
 
 ## Examples

docs/peer-validation.md

@@ -0,0 +1,37 @@
+# Peer Validation
+
+A [ValidatingAdmissionWebhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook) can  be used to avoid applying faulty Peer configurations to the cluster.
+
+## How It Works
+
+A [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly) is a Kubernetes resource that can be used to dynamically specify a service (i.e. the webhook server) that should validate operations (e.g. `UPDATE`, `CREATE`, etc.) on a particular resource (e.g. Kilo Peers).
+Once such a configuration is applied, the Kubernetes API server will send an AdmissionReviewRequest to the webhook service every time the specified operations are applied to the resource of the specified type.
+With regard to the [failure policy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy), the API server will apply the requested changes to a resource if the request was answered with `"allowed": true`, or deny the changes if the answer was `"allowed": false`.
+
+In case of Kilo Peer Validation, the specified operations are `UPDATE` and `CREATE`, the resources are `Peers`, and the default `failurePolicy` is set to `Fail`.
+View the full ValidatingWebhookConfiguration [here](https://github.com/leonnicolas/kilo-peer-validation/blob/main/deployment-no-cabundle.yaml).
+
+## Getting Started
+
+[Kilo-Peer-Validation](https://github.com/leonnicolas/kilo-peer-validation) is a webserver that rejects any AdmissionReviewRequest with a faulty Peer configuration.
+
+Apply the Service, the Deployment of the actual webserver, and the ValidatingWebhookConfiguration with:
+```shell
+kubectl apply -f https://raw.githubusercontent.com/leonnicolas/kilo-peer-validation/main/deployment-no-cabundle.yaml
+```
+
+The Kubernetes API server will only talk to webhook servers via TLS so the Kilo-Peer-Validation server must be given a valid TLS certificate and key, and the API server must be told what certificate authority (CA) to trust.
+One way to do this is to use the [kube-webhook-certgen](https://github.com/jet/kube-webhook-certgen) project to create a Kubernetes Secret holding the TLS certificate and key for the webhook server and to make a certificate signing request to the Kubernetes API server.
+The following snippet can be used to run kube-webhook-certgen in a Docker container to create a Secret and certificate signing request:
+```shell
+docker run -v /path/to/kubeconfig:/kubeconfig.yaml:ro jettech/kube-webhook-certgen:v1.5.2 --kubeconfig /kubeconfig.yaml create --namespace kilo --secret-name peer-validation-webhook-tls --host peer-validation,peer-validation.kilo.svc --key-name tls.key --cert-name tls.config
+```
+
+Now, the Kubernetes API server can be told what CA to trust by patching the ValidatingWebhookConfiguration with the newly created CA bundle:
+```shell
+docker run -v /path/to/kubeconfig:/kubeconfig.yaml:ro jettech/kube-webhook-certgen:v1.5.2 --kubeconfig /kubeconfig.yaml patch --webhook-name peer-validation.kilo.svc --secret-name peer-validation-webhook-tls --namespace kilo --patch-mutating=false
+```
+
+## Alternative Method
+
+An alternative method to generate a ValidatingWebhookConfiguration manifest without using Kubernetes' Certificate Signing API is described in [Kilo-Peer-Validation](https://github.com/leonnicolas/kilo-peer-validation#use-the-set-up-script).

docs/topology.md

@@ -11,9 +11,10 @@ This allows the encrypted network to serve several purposes, for example:
 
 By default, Kilo creates a mesh between the different logical locations in the cluster, e.g. data-centers, cloud providers, etc.
 Kilo will try to infer the location of the node using the [topology.kubernetes.io/region](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#topologykubernetesioregion) node label.
+Additionally, Kilo supports using a custom topology label by setting the command line flag `--topology-label=<label>`. 
 If this label is not set, then the [kilo.squat.ai/location](./annotations.md#location) node annotation can be used.
 
-For example, in order to join nodes in Google Cloud and AWS into a single cluster, an administrator could use the following snippet could to annotate all nodes with `GCP` in the name:
+For example, in order to join nodes in Google Cloud and AWS into a single cluster, an administrator could use the following snippet to annotate all nodes with `GCP` in the name:
 
 ```shell
 for node in $(kubectl get nodes | grep -i gcp | awk '{print $1}'); do kubectl annotate node $node kilo.squat.ai/location="gcp"; done

docs/userspace-wireguard.md

@@ -0,0 +1,44 @@
+# Userspace WireGuard
+
+It is possible to use a userspace implementation of WireGuard with Kilo.
+This can make sense in cases where
+
+* not all nodes in a cluster have WireGuard installed; or
+* nodes are effectively immutable and kernel modules cannot be installed.
+
+One example of a userspace implementation of WireGuard is [BoringTun].
+
+## Homogeneous Clusters
+
+In a homogeneous cluster where no node has the WireGuard kernel module, a userspace WireGuard implementation can be made available by deploying a DaemonSet.
+This DaemonSet creates a WireGuard interface that Kilo will manage.
+
+> **Note**: in order to avoid race conditions, `kg` needs to be passed the `--create-interface=false` flag. 
+
+An example configuration for a K3s cluster with [BoringTun] can be applied with:
+
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s-userspace.yaml
+```
+
+> **Note**: even if some nodes have the WireGuard kernel module, this configuration will cause all nodes to use the userspace implementation of WireGuard.
+
+## Heterogeneous Clusters
+
+In a heterogeneous cluster where some nodes are missing the WireGuard kernel module, a userspace WireGuard implementation can be provided only to the nodes that need it while enabling the other nodes to leverage WireGuard via the kernel module.
+An example of such a configuration for a K3s cluster can by applied with:
+
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s-userspace-heterogeneous.yaml
+```
+
+This configuration will deploy [nkml](https://github.com/leonnicolas/nkml) as a DaemonSet to label all nodes according to the presence of the WireGuard kernel module.
+It will also create two different DaemonSets with Kilo:
+1. `kilo` without userspace WireGuard; and
+1. `kilo-userspace` with [BoringTun] as a sidecar.
+
+> **Note**: because Kilo is dependant on nkml, nkml must be run on the host network before CNI is available and requires a kubeconfig in order to access the Kubernetes API.
+
+[BoringTun]: https://github.com/cloudflare/boringtun

docs/vpn.md

@@ -3,6 +3,7 @@
 Kilo enables peers outside of a Kubernetes cluster to connect to the created WireGuard network.
 This enables several use cases, for example:
 * giving cluster applications secure access to external services, e.g. services behind a corporate VPN;
+* improving the development flow of applications by running them locally and connecting them to the cluster;
 * allowing external services to access the cluster; and
 * enabling developers and support to securely debug cluster resources.
 
@@ -74,7 +75,7 @@ From any node or Pod on the cluster, one can now ping the peer:
 ping 10.5.0.1
 ```
 
-If the peer exposes a layer 4 service, for example an HTTP service, then one could also make requests against that endpoint from the cluster:
+If the peer exposes a layer 4 service, for example an HTTP server listening on TCP port 80, then one could also make requests against that endpoint from the cluster:
 
 ```shell
 curl http://10.5.0.1

e2e/full-mesh.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_full_mesh_connectivity() {
+	assert "retry 30 5 '' check_ping" "should be able to ping all Pods"
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings"
+	echo "sleep for 30s (one reconciliation period) and try again..."
+	sleep 30
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling"
+}
+
+test_full_mesh_peer() {
+	check_peer wg1 e2e 10.5.0.1/32 full
+}
+
+test_full_mesh_allowed_location_ips() {
+	docker exec kind-cluster-kilo-control-plane ip address add 10.6.0.1/32 dev eth0
+	_kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips=10.6.0.1/32
+	assert_equals Unauthorized "$(retry 10 5 'IP is not yet routable' curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz)" "should be able to make HTTP request to allowed location IP"
+	_kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips-
+	assert "retry 10 5 'IP is still routable' _not curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz" "should not be able to make HTTP request to allowed location IP"
+	docker exec kind-cluster-kilo-control-plane ip address delete 10.6.0.1/32 dev eth0
+}
+
+test_reject_peer_empty_allowed_ips() {
+	assert_fail "create_peer e2e '' 0 foo" "should not be able to create Peer with empty allowed IPs"
+}
+
+test_reject_peer_empty_public_key() {
+	assert_fail "create_peer e2e 10.5.0.1/32 0 ''" "should not be able to create Peer with empty public key"
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity full)"
+}

e2e/helper-curl.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+  labels:
+    app.kubernetes.io/name: curl
+  name: curl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: curl
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: curl
+    spec:
+      containers:
+      - command:
+        - /bin/sh
+        - -c
+        - while [ 1 -eq 1 ] ; do sleep 10; done
+        image: curlimages/curl
+        name: curl
+      restartPolicy: Always

e2e/kilo-kind-userspace.yaml

@@ -0,0 +1,204 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.3.1",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - update
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:test
+        imagePullPolicy: Never
+        args:
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --mesh-granularity=full
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+      - name: boringtun
+        image: leonnicolas/boringtun:alpine
+        args:
+        - --disable-drop-privileges=true
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo:test
+        imagePullPolicy: Never
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig

e2e/kind-config.yaml

@@ -0,0 +1,10 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane$WORKERS
+networking:
+  disableDefaultCNI: true # disable kindnet
+  apiServerAddress: 172.18.0.1
+  apiServerPort: $API_SERVER_PORT
+  podSubnet: $POD_SUBNET
+  serviceSubnet: $SERVICE_SUBNET

e2e/lib.sh

@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+KUBECONFIG="kind.yaml"
+KIND_CLUSTER="kind-cluster-kilo"
+KIND_BINARY="${KIND_BINARY:-kind}"
+KUBECTL_BINARY="${KUBECTL_BINARY:-kubectl}"
+KGCTL_BINARY="${KGCTL_BINARY:-kgctl}"
+KILO_IMAGE="${KILO_IMAGE:-squat/kilo}"
+
+retry() {
+	local COUNT="${1:-10}"
+	local SLEEP="${2:-5}"
+	local ERROR=$3
+	[ -n "$ERROR" ] && ERROR="$ERROR "
+	shift 3
+	for c in $(seq 1 "$COUNT"); do
+		if "$@"; then
+			return 0
+		else
+			printf "%s(attempt %d/%d)\n" "$ERROR" "$c" "$COUNT" | color "$YELLOW" 1>&2
+			if [ "$c" != "$COUNT" ]; then
+				printf "retrying in %d seconds...\n" "$SLEEP" | color "$YELLOW" 1>&2
+				sleep "$SLEEP"
+			fi
+		fi
+	done
+	return 1
+}
+
+_not() {
+	if "$@"; then
+		return 1
+	fi
+	return 0
+}
+
+# _kubectl is a helper that calls kubectl with the --kubeconfig flag.
+_kubectl() {
+	$KUBECTL_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# _kgctl is a helper that calls kgctl with the --kubeconfig flag.
+_kgctl() {
+	$KGCTL_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# _kind is a helper that calls kind with the --kubeconfig flag.
+_kind() {
+	$KIND_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# shellcheck disable=SC2120
+build_kind_config() {
+	local WORKER_COUNT="${1:-0}"
+	export API_SERVER_PORT="${2:-6443}"
+	export POD_SUBNET="${3:-10.42.0.0/16}"
+	export SERVICE_SUBNET="${4:-10.43.0.0/16}"
+	export WORKERS="" 
+	local i=0
+	while [ "$i" -lt "$WORKER_COUNT" ]; do
+		WORKERS="$(printf "%s\n- role: worker" "$WORKERS")"
+		((i++))
+	done
+	envsubst < ./kind-config.yaml
+	unset API_SERVER_PORT POD_SUBNET SERVICE_SUBNET WORKERS
+}
+
+create_interface() {
+	docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun --foreground --disable-drop-privileges true "$1"
+}
+
+delete_interface() {
+	docker rm --force "$1"
+}
+
+create_peer() {
+	cat <<EOF | _kubectl apply -f -
+apiVersion: kilo.squat.ai/v1alpha1
+kind: Peer
+metadata:
+  name: $1
+spec:
+  allowedIPs:
+  - $2
+  persistentKeepalive: $3
+  publicKey: $4
+EOF
+}
+
+delete_peer() {
+	_kubectl delete peer "$1"
+}
+
+is_ready() {
+	for pod in $(_kubectl -n "$1" get pods -o name -l "$2"); do
+		if ! _kubectl -n "$1" get "$pod" | tail -n 1 | grep -q Running; then
+			return 1;
+		fi
+	done
+	return 0
+}
+
+# Returns non zero if one pod of the given name in the given namespace is not ready.
+block_until_ready_by_name() {
+	block_until_ready "$1" "app.kubernetes.io/name=$2"
+}
+
+# Blocks until all pods of a deployment are ready.
+block_until_ready() {
+	retry 30 5 "some $2 pods are not ready yet" is_ready "$1" "$2"
+}
+
+
+# create_cluster launches a kind cluster and deploys Kilo, Adjacency, and a helper with curl.
+create_cluster() {
+	# shellcheck disable=SC2119
+	local CONFIG="${1:-$(build_kind_config)}"
+	_kind delete clusters $KIND_CLUSTER > /dev/null
+	# Create the kind cluster.
+	_kind create cluster --name $KIND_CLUSTER --config <(echo "$CONFIG")
+	# Load the Kilo image into kind.
+	docker tag "$KILO_IMAGE" squat/kilo:test
+	# This command does not accept the --kubeconfig flag, so call the command directly.
+	$KIND_BINARY load docker-image squat/kilo:test --name $KIND_CLUSTER
+	# Create the kubeconfig secret.
+	_kubectl create secret generic kubeconfig --from-file=kubeconfig="$KUBECONFIG" -n kube-system
+	# Apply Kilo the the cluster.
+	_kubectl apply -f ../manifests/crds.yaml
+	_kubectl apply -f kilo-kind-userspace.yaml
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait nodes --all --for=condition=Ready
+	# Wait for CoreDNS.
+	block_until_ready kube_system k8s-app=kube-dns
+	# Ensure the curl helper is not scheduled on a control-plane node.
+	_kubectl apply -f helper-curl.yaml
+	block_until_ready_by_name default curl
+	_kubectl taint node $KIND_CLUSTER-control-plane node-role.kubernetes.io/master:NoSchedule-
+	_kubectl apply -f https://raw.githubusercontent.com/heptoprint/adjacency/master/example.yaml
+	block_until_ready_by_name adjacency adjacency
+}
+
+delete_cluster () {
+	_kind delete clusters $KIND_CLUSTER
+}
+
+curl_pod() {
+	_kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /usr/bin/curl "$@"
+}
+
+check_ping() {
+	local LOCAL
+	while [ $# -gt 0 ]; do
+		case $1 in
+			--local)
+			LOCAL=true
+			;;
+		esac
+		shift
+	done
+
+	for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		if [ -n "$LOCAL" ]; then
+			ping=$(curl -m 1 -s http://"$ip":8080/ping)
+		else
+			ping=$(curl_pod -m 1 -s http://"$ip":8080/ping)
+		fi
+		if [ "$ping" = "pong" ]; then
+			echo "successfully pinged $ip"
+		else
+			printf 'failed to ping %s; expected "pong" but got "%s"\n' "$ip" "$ping"
+			return 1
+		fi
+	done
+	return 0
+}
+
+check_adjacent() {
+	curl_pod adjacency:8080/?format=fancy
+	[ "$(curl_pod -m 1 -s adjacency:8080/?format=json | jq '.[].latencies[].ok' | grep -c true)" -eq $(($1*$1)) ]
+}
+
+check_peer() {
+	local INTERFACE=$1
+	local PEER=$2
+	local ALLOWED_IP=$3
+	local GRANULARITY=$4
+	create_interface "$INTERFACE"
+	docker run --rm --entrypoint=/usr/bin/wg "$KILO_IMAGE" genkey > "$INTERFACE"
+	assert "create_peer $PEER $ALLOWED_IP 10 $(docker run --rm --entrypoint=/bin/sh -v "$PWD/$INTERFACE":/key "$KILO_IMAGE" -c 'cat /key | wg pubkey')" "should be able to create Peer"
+	assert "_kgctl showconf peer $PEER --mesh-granularity=$GRANULARITY > $PEER.ini" "should be able to get Peer configuration"
+	assert "docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v $PWD/$PEER.ini:/peer.ini $KILO_IMAGE setconf $INTERFACE /peer.ini" "should be able to apply configuration from kgctl"
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v "$PWD/$INTERFACE":/key "$KILO_IMAGE" set "$INTERFACE" private-key /key
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" address add "$ALLOWED_IP" dev "$INTERFACE"
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" link set "$INTERFACE" up
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" route add 10.42/16 dev "$INTERFACE"
+	assert "retry 10 5 '' check_ping --local" "should be able to ping Pods from host"
+	assert_equals "$(_kgctl showconf peer "$PEER")" "$(_kgctl showconf peer "$PEER" --mesh-granularity="$GRANULARITY")" "kgctl should be able to auto detect the mesh granularity"
+	rm "$INTERFACE" "$PEER".ini
+	delete_peer "$PEER"
+	delete_interface "$INTERFACE"
+}

e2e/location-mesh.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_location_mesh_connectivity() {
+	assert "retry 30 5 '' check_ping" "should be able to ping all Pods"
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings"
+	echo "sleep for 30s (one reconciliation period) and try again..."
+	sleep 30
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling"
+}
+
+test_location_mesh_peer() {
+	check_peer wg1 e2e 10.5.0.1/32 location
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity location)"
+}

e2e/multi-cluster.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+# shellcheck disable=SC2153
+KUBECONFIG2="$KUBECONFIG"2
+# shellcheck disable=SC2153
+KIND_CLUSTER2="$KIND_CLUSTER"2
+
+setup_suite() {
+	KUBECONFIG=$KUBECONFIG2 KIND_CLUSTER=$KIND_CLUSTER2 create_cluster "$(build_kind_config 1 6444 10.44.0.0/16 10.45.0.0/16)"
+	# shellcheck disable=SC2016
+	KUBECONFIG=$KUBECONFIG2 _kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full","--subnet=10.6.0.0/16"]}]}}}}'
+	KUBECONFIG=$KUBECONFIG2 block_until_ready_by_name kube-system kilo-userspace 
+	# Register the nodes in cluster1 as peers of cluster2.
+	for n in $(_kubectl get no -o name | cut -d'/' -f2); do
+		# Specify the service CIDR as an extra IP range that should be routable.
+		$KGCTL_BINARY --kubeconfig "$KUBECONFIG" showconf node "$n" --as-peer -o yaml --allowed-ips 10.43.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG2" apply -f -
+	done
+	# Register the nodes in cluster2 as peers of cluster1.
+	for n in $(KUBECONFIG=$KUBECONFIG2 _kubectl get no -o name | cut -d'/' -f2); do
+		# Specify the service CIDR as an extra IP range that should be routable.
+		$KGCTL_BINARY --kubeconfig "$KUBECONFIG2" showconf node "$n" --as-peer -o yaml --allowed-ips 10.45.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG" apply -f -
+	done
+}
+
+test_multi_cluster_pod_connectivity() {
+	for ip in $(KUBECONFIG=$KUBECONFIG2 _kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		assert_equals pong "$(retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 1 to Pod in cluster 2"
+	done
+	for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		assert_equals pong "$(KUBECONFIG="$KUBECONFIG2" retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 2 to Pod in cluster 1"
+	done
+}
+
+test_multi_cluster_service_connectivity() {
+	# Mirror the Kubernetes API service from cluster1 into cluster2.
+	cat <<EOF | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG2" apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: mirrored-kubernetes
+spec:
+  ports:
+    - port: 443
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+    name: mirrored-kubernetes
+subsets:
+  - addresses:
+      - ip: $(_kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}') # The cluster IP of the Kubernetes API service on cluster1.
+    ports:
+      - port: 443
+EOF
+        assert_equals ok "$(KUBECONFIG="$KUBECONFIG2" retry 10 5 "service is not yet routable" curl_pod -m 1 -s -k https://mirrored-kubernetes/readyz)" "should be able to make HTTP request from cluster 2 to service in cluster 1"
+}
+
+teardown_suite () {
+	# Remove the nodes in cluster2 as peers of cluster1.
+	for n in $(KUBECONFIG=$KUBECONFIG2 _kubectl get no -o name | cut -d'/' -f2); do
+		_kubectl delete peer "$n"
+	done
+	KUBECONFIG=$KUBECONFIG2 KIND_CLUSTER=$KIND_CLUSTER2 delete_cluster
+}

e2e/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	create_cluster "$(build_kind_config 2)"
+}

e2e/teardown.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+teardown_suite () {
+	delete_cluster
+}

go.mod

@@ -1,64 +1,28 @@
 module github.com/squat/kilo
 
-go 1.14
+go 1.15
 
 require (
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/ant31/crd-validation v0.0.0-20180801212718-38f6a293f140
 	github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310
+	github.com/campoy/embedmd v1.0.0
 	github.com/containernetworking/cni v0.6.0
 	github.com/containernetworking/plugins v0.6.0
 	github.com/coreos/go-iptables v0.4.0
-	github.com/emicklei/go-restful v2.9.3+incompatible // indirect
+	github.com/go-kit/kit v0.9.0
-	github.com/evanphx/json-patch v4.2.0+incompatible // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-kit/kit v0.8.0
-	github.com/go-logfmt/logfmt v0.4.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.0 // indirect
-	github.com/go-openapi/jsonreference v0.19.0 // indirect
-	github.com/go-openapi/spec v0.19.0
-	github.com/go-openapi/swag v0.19.0 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect
-	github.com/golang/protobuf v1.3.1 // indirect
-	github.com/google/gofuzz v1.0.0 // indirect
-	github.com/googleapis/gnostic v0.2.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
-	github.com/json-iterator/go v1.1.6 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
-	github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/oklog/run v1.0.0
-	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/prometheus/client_golang v1.7.1
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/spf13/cobra v1.1.3
-	github.com/prometheus/client_golang v0.9.2
+	github.com/spf13/pflag v1.0.5
-	github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267
-	github.com/stretchr/testify v1.3.0 // indirect
 	github.com/vishvananda/netlink v1.0.0
 	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
-	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
-	golang.org/x/lint v0.0.0-20190409202823-959b441ac422
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
-	golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 // indirect
+	k8s.io/api v0.21.1
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
+	k8s.io/apiextensions-apiserver v0.21.1
-	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872
+	k8s.io/apimachinery v0.21.1
-	golang.org/x/text v0.3.2 // indirect
+	k8s.io/client-go v0.21.1
-	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
+	k8s.io/code-generator v0.21.1
-	golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 // indirect
+	sigs.k8s.io/controller-tools v0.6.0
-	google.golang.org/appengine v1.5.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	k8s.io/api v0.0.0-20190313235455-40a48860b5ab
-	k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed
-	k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1
-	k8s.io/client-go v11.0.0+incompatible
-	k8s.io/code-generator v0.0.0-20190311093542-50b561225d70
-	k8s.io/gengo v0.0.0-20181106084056-51747d6e00da // indirect
-	k8s.io/klog v0.3.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30
-	k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 // indirect
-	sigs.k8s.io/yaml v1.1.0 // indirect
 )

go.sum

@@ -1,218 +1,737 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
+github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/ant31/crd-validation v0.0.0-20180801212718-38f6a293f140 h1:ljF9TLBK+JYXKB8aklLhcubPDBUjpvzWKiqgZ37s+QY=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/ant31/crd-validation v0.0.0-20180801212718-38f6a293f140/go.mod h1:X0noFIik9YqfhGYBLEHg8LJKEwy7QIitLQuFMpKLcPk=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U=
 github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
+github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containernetworking/cni v0.6.0 h1:FXICGBZNMtdHlW65trpoHviHctQD3seWhRRcqp2hMOU=
 github.com/containernetworking/cni v0.6.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/plugins v0.6.0 h1:bqPT7yYisnWs+FrtgY5/qLEB9QZ/6z11wMNCwSdzZm0=
 github.com/containernetworking/plugins v0.6.0/go.mod h1:dagHaAhNjXjT9QYOklkKJDGaQPTg4pf//FrUcJeb7FU=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.4.0 h1:wh4UbVs8DhLUbpyq97GLJDKrQMjEDD63T1xE4CrsKzQ=
 github.com/coreos/go-iptables v0.4.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emicklei/go-restful v2.9.3+incompatible h1:2OwhVdhtzYUp5P5wuGsVDPagKSRd9JK72sJCHVCXh5g=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/emicklei/go-restful v2.9.3+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=
+github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-openapi/jsonpointer v0.19.0 h1:FTUMcX77w5rQkClIzDtTxvn6Bsa894CcrzNj2MMfeg8=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-openapi/jsonreference v0.17.0 h1:yJW3HCkTHg7NOA+gZ83IPHzUSnUzGXhGmsdiCcMexbA=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.19.0 h1:A4SZ6IWh3lnjH0rG0Z5lkxazMGBECtrZcbyYQi+64k4=
+github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
-github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi880=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.0 h1:Kg7Wl7LkTPlmc393QZQ/5rQadPhi7pBVEMZxyTi0Ii8=
-github.com/go-openapi/swag v0.19.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gobuffalo/flect v0.2.2 h1:PAVD7sp0KOdfswjAw9BpLCU9hXo7wFSzgpQ+zNeks/A=
+github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
+github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267 h1:I9j1PLS64+NgCtkgbomGInboj1NFH1KF1tkVKlt3yF4=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
 github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58 h1:otZG8yDCO4LVps5+9bxOeNiCvgmOyt96J3roHTYs7oE=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYRuq8JQ1aa7LJt8EXVyo=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-k8s.io/api v0.0.0-20190313235455-40a48860b5ab h1:DG9A67baNpoeweOy2spF1OWHhnVY5KR7/Ek/+U1lVZc=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed h1:rCteec//ELIjZMfjIGQbVtZooyaofqDJwsmWwWKItNs=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 h1:IS7K02iBkQXpCeieSiyJjGoLSdVOv2DbPaWHJ+ZtgKg=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-k8s.io/code-generator v0.0.0-20190311093542-50b561225d70 h1:lgPp615xLHxN84RBd+viA/oHzJfI0miFYFH4T9wpPQ4=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/code-generator v0.0.0-20190311093542-50b561225d70/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-k8s.io/gengo v0.0.0-20181106084056-51747d6e00da h1:ZMvcXtMVbhUCtCuiSEzBV+Eur4swzfdxx6ZyX3qT6dk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 h1:8r+l4bNWjRlsFYlQJnKJ2p7s1YQPj4XyXiJVqDHRx7c=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
+k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c=
-sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
+k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo=
+k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA=
+k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs=
+k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
+k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY=
+k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4=
+k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
+k8s.io/code-generator v0.21.1 h1:jvcxHpVu5dm/LMXr3GOj/jroiP8+v2YnJE9i2OVRenk=
+k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
+k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
+k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/controller-tools v0.6.0 h1:o2Fm1K7CmIp8OVaBtXsWB/ssBAzyoKZPPAGR3VuxaKs=
+sigs.k8s.io/controller-tools v0.6.0/go.mod h1:baRMVPrctU77F+rfAuH2uPqW93k6yQnZA2dhUOr7ihc=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

manifests/crds.yaml

@@ -0,0 +1,92 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0
+  creationTimestamp: null
+  name: peers.kilo.squat.ai
+spec:
+  group: kilo.squat.ai
+  names:
+    kind: Peer
+    listKind: PeerList
+    plural: peers
+    singular: peer
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Peer is a WireGuard peer that should have access to the VPN.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: 'Specification of the desired behavior of the Kilo Peer.
+              More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status'
+            properties:
+              allowedIPs:
+                description: AllowedIPs is the list of IP addresses that are allowed
+                  for the given peer's tunnel.
+                items:
+                  type: string
+                type: array
+              endpoint:
+                description: Endpoint is the initial endpoint for connections to the
+                  peer.
+                properties:
+                  dnsOrIP:
+                    description: DNSOrIP is a DNS name or an IP address.
+                    properties:
+                      dns:
+                        description: DNS must be a valid RFC 1123 subdomain.
+                        type: string
+                      ip:
+                        description: IP must be a valid IP address.
+                        type: string
+                    type: object
+                  port:
+                    description: Port must be a valid port number.
+                    format: int32
+                    type: integer
+                required:
+                - dnsOrIP
+                - port
+                type: object
+              persistentKeepalive:
+                description: PersistentKeepalive is the interval in seconds of the
+                  emission of keepalive packets by the peer. This defaults to 0, which
+                  disables the feature.
+                type: integer
+              presharedKey:
+                description: PresharedKey is the optional symmetric encryption key
+                  for the peer.
+                type: string
+              publicKey:
+                description: PublicKey is the WireGuard public key for the peer.
+                type: string
+            required:
+            - allowedIPs
+            - publicKey
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

manifests/kilo-bootkube-flannel.yaml

@@ -30,7 +30,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,14 +52,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -77,6 +80,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-bootkube.yaml

@@ -64,7 +64,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,14 +86,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -108,6 +111,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-k3s-flannel.yaml

@@ -30,7 +30,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,14 +52,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -77,6 +80,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-k3s-userspace-heterogeneous.yaml

@@ -0,0 +1,361 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.3.1",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - get
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - update
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      nodeSelector:
+        nkml.squat.ai/wireguard: "true"
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes/kubeconfig
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        hostPath: 
+          # Since kilo runs as a daemonset, it is recommended that you copy the
+          # k3s.yaml kubeconfig file from the master node to all worker nodes
+          # with the same path structure.
+          path: /etc/rancher/k3s/k3s.yaml
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo-userspace
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      nodeSelector:
+        nkml.squat.ai/wireguard: "false"
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo 
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes/kubeconfig
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      - name: boringtun
+        image: leonnicolas/boringtun
+        args:
+        - --disable-drop-privileges=true
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        hostPath: 
+          # Since kilo runs as a daemonset, it is recommended that you copy the
+          # k3s.yaml kubeconfig file from the master node to all worker nodes
+          # with the same path structure.
+          path: /etc/rancher/k3s/k3s.yaml
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata: 
+  name: nkml
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: nkml
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nkml
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nkml
+    spec:
+      hostNetwork: true
+      containers:
+      - name: nkml
+        image: leonnicolas/nkml
+        args: 
+        - --hostname=$(NODE_NAME)
+        - --label-mod=wireguard
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - name: http
+          containerPort: 8080
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes/kubeconfig
+          readOnly: true
+      volumes:
+      - name: kubeconfig
+        hostPath: 
+          # since the above DaemonSets are dependant on the labels
+          # and nkml would need a cni to start
+          # it needs run on the hostnetwork and use the kubeconfig
+          # to label the nodes
+          path: /etc/rancher/k3s/k3s.yaml

manifests/kilo-k3s-userspace.yaml

@@ -0,0 +1,205 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.3.1",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - update
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes/kubeconfig
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      - name: boringtun
+        image: leonnicolas/boringtun
+        args:
+        - --disable-drop-privileges=true
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        hostPath: 
+          # Since kilo runs as a daemonset, it is recommended that you copy the
+          # k3s.yaml kubeconfig file from the master node to all worker nodes
+          # with the same path structure.
+          path: /etc/rancher/k3s/k3s.yaml
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard

manifests/kilo-k3s.yaml

@@ -64,7 +64,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,14 +86,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -108,6 +111,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-kubeadm-flannel.yaml

@@ -30,7 +30,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,14 +52,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -77,6 +80,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-kubeadm.yaml

@@ -64,7 +64,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,14 +86,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -108,6 +111,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -160,7 +166,7 @@ spec:
           path: /opt/cni/bin
       - name: cni-conf-dir
         hostPath:
-          path: /etc/kubernetes/cni/net.d
+          path: /etc/cni/net.d
       - name: kilo-dir
         hostPath:
           path: /var/lib/kilo

manifests/kilo-typhoon-flannel.yaml

@@ -30,7 +30,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,14 +52,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -77,6 +80,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-typhoon.yaml

@@ -64,7 +64,7 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,14 +86,17 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
@@ -108,6 +111,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/podmonitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+  name: kilo
+  namespace: kilo
+spec:
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  podMetricsEndpoints:
+  - interval: 15s
+    port: metrics
+    path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: kilo

pkg/encapsulation/ipip.go

@@ -67,17 +67,18 @@ func (i *ipip) Init(base int) error {
 // when traffic between nodes must be encapsulated.
 func (i *ipip) Rules(nodes []*net.IPNet) []iptables.Rule {
 	var rules []iptables.Rule
+	proto := ipipProtocolName()
 	rules = append(rules, iptables.NewIPv4Chain("filter", "KILO-IPIP"))
 	rules = append(rules, iptables.NewIPv6Chain("filter", "KILO-IPIP"))
-	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-p", "4", "-j", "KILO-IPIP"))
+	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-j", "KILO-IPIP"))
-	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-p", "4", "-j", "KILO-IPIP"))
+	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-j", "KILO-IPIP"))
 	for _, n := range nodes {
 		// Accept encapsulated traffic from peers.
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(n.IP)), "filter", "KILO-IPIP", "-m", "comment", "--comment", "Kilo: allow IPIP traffic", "-s", n.IP.String(), "-j", "ACCEPT"))
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(n.IP)), "filter", "KILO-IPIP", "-s", n.String(), "-m", "comment", "--comment", "Kilo: allow IPIP traffic", "-j", "ACCEPT"))
 	}
 	// Drop all other IPIP traffic.
-	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-p", "4", "-j", "DROP"))
+	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-j", "DROP"))
-	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-p", "4", "-j", "DROP"))
+	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-j", "DROP"))
 
 	return rules
 }

pkg/encapsulation/ipip_cgo.go

@@ -1,4 +1,4 @@
-// Copyright 2015 go-swagger maintainers
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,12 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package spec
+// +build cgo
 
-// License information for the exposed API.
+package encapsulation
-//
+
-// For more information: http://goo.gl/8us55a#licenseObject
+/*
-type License struct {
+#include <netdb.h>
-	Name string `json:"name,omitempty"`
+*/
-	URL  string `json:"url,omitempty"`
+import "C"
+
+func ipipProtocolName() string {
+	return C.GoString(C.getprotobynumber(4).p_name)
 }

pkg/encapsulation/ipip_nocgo.go

@@ -1,4 +1,4 @@
-// Copyright 2015 go-swagger maintainers
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package spec
+// +build !cgo
 
-// ContactInfo contact information for the exposed API.
+package encapsulation
-//
+
-// For more information: http://goo.gl/8us55a#contactObject
+// If we can determine the protocol name at runtime
-type ContactInfo struct {
+// by looking it up in the protocols database, assume `ipencap`
-	Name  string `json:"name,omitempty"`
+// as this is the value in Kilo's container image.
-	URL   string `json:"url,omitempty"`
+func ipipProtocolName() string {
-	Email string `json:"email,omitempty"`
+	return "ipencap"
 }

pkg/encapsulation/noop.go

@@ -0,0 +1,59 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encapsulation
+
+import (
+	"net"
+
+	"github.com/squat/kilo/pkg/iptables"
+)
+
+// Noop is an encapsulation that does nothing.
+type Noop Strategy
+
+// CleanUp will also do nothing.
+func (n Noop) CleanUp() error {
+	return nil
+}
+
+// Gw will also do nothing.
+func (n Noop) Gw(_ net.IP, _ net.IP, _ *net.IPNet) net.IP {
+	return nil
+}
+
+// Index will also do nothing.
+func (n Noop) Index() int {
+	return 0
+}
+
+// Init will also do nothing.
+func (n Noop) Init(_ int) error {
+	return nil
+}
+
+// Rules will also do nothing.
+func (n Noop) Rules(_ []*net.IPNet) []iptables.Rule {
+	return nil
+}
+
+// Set will also do nothing.
+func (n Noop) Set(_ *net.IPNet) error {
+	return nil
+}
+
+// Strategy will finally do nothing.
+func (n Noop) Strategy() Strategy {
+	return Strategy(n)
+}

pkg/iptables/fake.go

@@ -16,6 +16,8 @@ package iptables
 
 import (
 	"fmt"
+	"strings"
+	"sync/atomic"
 
 	"github.com/coreos/go-iptables/iptables"
 )
@@ -38,12 +40,14 @@ func (s statusError) ExitStatus() int {
 }
 
 type fakeClient struct {
+	calls   uint64
 	storage []Rule
 }
 
 var _ Client = &fakeClient{}
 
 func (f *fakeClient) AppendUnique(table, chain string, spec ...string) error {
+	atomic.AddUint64(&f.calls, 1)
 	exists, err := f.Exists(table, chain, spec...)
 	if err != nil {
 		return err
@@ -56,6 +60,7 @@ func (f *fakeClient) AppendUnique(table, chain string, spec ...string) error {
 }
 
 func (f *fakeClient) Delete(table, chain string, spec ...string) error {
+	atomic.AddUint64(&f.calls, 1)
 	r := &rule{table: table, chain: chain, spec: spec}
 	for i := range f.storage {
 		if f.storage[i].String() == r.String() {
@@ -69,6 +74,7 @@ func (f *fakeClient) Delete(table, chain string, spec ...string) error {
 }
 
 func (f *fakeClient) Exists(table, chain string, spec ...string) (bool, error) {
+	atomic.AddUint64(&f.calls, 1)
 	r := &rule{table: table, chain: chain, spec: spec}
 	for i := range f.storage {
 		if f.storage[i].String() == r.String() {
@@ -78,7 +84,22 @@ func (f *fakeClient) Exists(table, chain string, spec ...string) (bool, error) {
 	return false, nil
 }
 
+func (f *fakeClient) List(table, chain string) ([]string, error) {
+	atomic.AddUint64(&f.calls, 1)
+	var rs []string
+	for i := range f.storage {
+		switch r := f.storage[i].(type) {
+		case *rule:
+			if r.table == table && r.chain == chain {
+				rs = append(rs, strings.TrimSpace(strings.TrimPrefix(r.String(), table)))
+			}
+		}
+	}
+	return rs, nil
+}
+
 func (f *fakeClient) ClearChain(table, name string) error {
+	atomic.AddUint64(&f.calls, 1)
 	for i := range f.storage {
 		r, ok := f.storage[i].(*rule)
 		if !ok {
@@ -90,10 +111,14 @@ func (f *fakeClient) ClearChain(table, name string) error {
 			}
 		}
 	}
-	return f.DeleteChain(table, name)
+	if err := f.DeleteChain(table, name); err != nil {
+		return err
+	}
+	return f.NewChain(table, name)
 }
 
 func (f *fakeClient) DeleteChain(table, name string) error {
+	atomic.AddUint64(&f.calls, 1)
 	for i := range f.storage {
 		r, ok := f.storage[i].(*rule)
 		if !ok {
@@ -116,6 +141,7 @@ func (f *fakeClient) DeleteChain(table, name string) error {
 }
 
 func (f *fakeClient) NewChain(table, name string) error {
+	atomic.AddUint64(&f.calls, 1)
 	c := &chain{table: table, chain: name}
 	for i := range f.storage {
 		if f.storage[i].String() == c.String() {
@@ -125,3 +151,17 @@ func (f *fakeClient) NewChain(table, name string) error {
 	f.storage = append(f.storage, c)
 	return nil
 }
+
+func (f *fakeClient) ListChains(table string) ([]string, error) {
+	atomic.AddUint64(&f.calls, 1)
+	var cs []string
+	for i := range f.storage {
+		switch c := f.storage[i].(type) {
+		case *chain:
+			if c.table == table {
+				cs = append(cs, c.chain)
+			}
+		}
+	}
+	return cs, nil
+}

pkg/iptables/iptables.go

@@ -17,11 +17,12 @@ package iptables
 import (
 	"fmt"
 	"net"
-	"strings"
 	"sync"
 	"time"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
 )
 
 // Protocol represents an IP protocol.
@@ -47,9 +48,11 @@ type Client interface {
 	AppendUnique(table string, chain string, rule ...string) error
 	Delete(table string, chain string, rule ...string) error
 	Exists(table string, chain string, rule ...string) (bool, error)
+	List(table string, chain string) ([]string, error)
 	ClearChain(table string, chain string) error
 	DeleteChain(table string, chain string) error
 	NewChain(table string, chain string) error
+	ListChains(table string) ([]string, error)
 }
 
 // Rule is an interface for interacting with iptables objects.
@@ -107,7 +110,17 @@ func (r *rule) String() string {
 	if r == nil {
 		return ""
 	}
-	return fmt.Sprintf("%s_%s_%s", r.table, r.chain, strings.Join(r.spec, "_"))
+	spec := r.table + " -A " + r.chain
+	for i, s := range r.spec {
+		spec += " "
+		// If this is the content of a comment, wrap the value in quotes.
+		if i > 0 && r.spec[i-1] == "--comment" {
+			spec += `"` + s + `"`
+		} else {
+			spec += s
+		}
+	}
+	return spec
 }
 
 func (r *rule) Proto() Protocol {
@@ -132,6 +145,7 @@ func NewIPv6Chain(table, name string) Rule {
 }
 
 func (c *chain) Add(client Client) error {
+	// Note: `ClearChain` creates a chain if it does not exist.
 	if err := client.ClearChain(c.table, c.chain); err != nil {
 		return fmt.Errorf("failed to add iptables chain: %v", err)
 	}
@@ -171,41 +185,81 @@ func (c *chain) String() string {
 	if c == nil {
 		return ""
 	}
-	return fmt.Sprintf("%s_%s", c.table, c.chain)
+	return chainToString(c.table, c.chain)
 }
 
 func (c *chain) Proto() Protocol {
 	return c.proto
 }
 
+func chainToString(table, chain string) string {
+	return fmt.Sprintf("%s -N %s", table, chain)
+}
+
 // Controller is able to reconcile a given set of iptables rules.
 type Controller struct {
 	v4           Client
 	v6           Client
 	errors       chan error
+	logger       log.Logger
+	resyncPeriod time.Duration
 
 	sync.Mutex
 	rules      []Rule
 	subscribed bool
 }
 
+// ControllerOption modifies the controller's configuration.
+type ControllerOption func(h *Controller)
+
+// WithLogger adds a logger to the controller.
+func WithLogger(logger log.Logger) ControllerOption {
+	return func(c *Controller) {
+		c.logger = logger
+	}
+}
+
+// WithResyncPeriod modifies how often the controller reconciles.
+func WithResyncPeriod(resyncPeriod time.Duration) ControllerOption {
+	return func(c *Controller) {
+		c.resyncPeriod = resyncPeriod
+	}
+}
+
+// WithClients adds iptables clients to the controller.
+func WithClients(v4, v6 Client) ControllerOption {
+	return func(c *Controller) {
+		c.v4 = v4
+		c.v6 = v6
+	}
+}
+
 // New generates a new iptables rules controller.
-// It expects an IP address length to determine
+// If no options are given, IPv4 and IPv6 clients
-// whether to operate in IPv4 or IPv6 mode.
+// will be instantiated using the regular iptables backend.
-func New() (*Controller, error) {
+func New(opts ...ControllerOption) (*Controller, error) {
+	c := &Controller{
+		errors: make(chan error),
+		logger: log.NewNopLogger(),
+	}
+	for _, o := range opts {
+		o(c)
+	}
+	if c.v4 == nil {
 		v4, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create iptables IPv4 client: %v", err)
 		}
+		c.v4 = v4
+	}
+	if c.v6 == nil {
 		v6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create iptables IPv6 client: %v", err)
 		}
-	return &Controller{
+		c.v6 = v6
-		v4:     v4,
+	}
-		v6:     v6,
+	return c, nil
-		errors: make(chan error),
-	}, nil
 }
 
 // Run watches for changes to iptables rules and reconciles
@@ -220,16 +274,18 @@ func (c *Controller) Run(stop <-chan struct{}) (<-chan error, error) {
 	c.subscribed = true
 	c.Unlock()
 	go func() {
+		t := time.NewTimer(c.resyncPeriod)
 		defer close(c.errors)
 		for {
 			select {
-			case <-time.After(5 * time.Second):
+			case <-t.C:
-			case <-stop:
-				return
-			}
 				if err := c.reconcile(); err != nil {
 					nonBlockingSend(c.errors, fmt.Errorf("failed to reconcile rules: %v", err))
 				}
+				t.Reset(c.resyncPeriod)
+			case <-stop:
+				return
+			}
 		}
 	}()
 	return c.errors, nil
@@ -242,12 +298,14 @@ func (c *Controller) Run(stop <-chan struct{}) (<-chan error, error) {
 func (c *Controller) reconcile() error {
 	c.Lock()
 	defer c.Unlock()
+	var rc ruleCache
 	for i, r := range c.rules {
-		ok, err := r.Exists(c.client(r.Proto()))
+		ok, err := rc.exists(c.client(r.Proto()), r)
 		if err != nil {
 			return fmt.Errorf("failed to check if rule exists: %v", err)
 		}
 		if !ok {
+			level.Info(c.logger).Log("msg", fmt.Sprintf("applying %d iptables rules", len(c.rules)-i))
 			if err := c.resetFromIndex(i, c.rules); err != nil {
 				return fmt.Errorf("failed to add rule: %v", err)
 			}

pkg/iptables/iptables_test.go

@@ -83,10 +83,11 @@ func TestSet(t *testing.T) {
 			},
 		},
 	} {
-		controller := &Controller{}
 		client := &fakeClient{}
-		controller.v4 = client
+		controller, err := New(WithClients(client, client))
-		controller.v6 = client
+		if err != nil {
+			t.Fatalf("test case %q: got unexpected error instantiating controller: %v", tc.name, err)
+		}
 		for i := range tc.sets {
 			if err := controller.Set(tc.sets[i]); err != nil {
 				t.Fatalf("test case %q: got unexpected error seting rule set %d: %v", tc.name, i, err)
@@ -139,10 +140,11 @@ func TestCleanUp(t *testing.T) {
 			rules: []Rule{rules[0], rules[1]},
 		},
 	} {
-		controller := &Controller{}
 		client := &fakeClient{}
-		controller.v4 = client
+		controller, err := New(WithClients(client, client))
-		controller.v6 = client
+		if err != nil {
+			t.Fatalf("test case %q: got unexpected error instantiating controller: %v", tc.name, err)
+		}
 		if err := controller.Set(tc.rules); err != nil {
 			t.Fatalf("test case %q: Set should not fail: %v", tc.name, err)
 		}

pkg/iptables/rulecache.go

@@ -0,0 +1,106 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ruleCacheFlag byte
+
+const (
+	exists ruleCacheFlag = 1 << iota
+	populated
+)
+
+type isNotExistError interface {
+	error
+	IsNotExist() bool
+}
+
+// ruleCache is a lazy cache that can be used to
+// check if a given rule or chain exists in an iptables
+// table.
+type ruleCache [2]map[string]ruleCacheFlag
+
+func (rc *ruleCache) populateTable(c Client, proto Protocol, table string) error {
+	// If the table already exists in the destination map,
+	// exit early since it has already been populated.
+	if rc[proto][table]&populated != 0 {
+		return nil
+	}
+	cs, err := c.ListChains(table)
+	if err != nil {
+		return fmt.Errorf("failed to populate chains for table %q: %v", table, err)
+	}
+	rc[proto][table] = exists | populated
+	for i := range cs {
+		rc[proto][chainToString(table, cs[i])] |= exists
+	}
+	return nil
+}
+
+func (rc *ruleCache) populateChain(c Client, proto Protocol, table, chain string) error {
+	// If the destination chain true, then it has already been populated.
+	if rc[proto][chainToString(table, chain)]&populated != 0 {
+		return nil
+	}
+	rs, err := c.List(table, chain)
+	if err != nil {
+		if existsErr, ok := err.(isNotExistError); ok && existsErr.IsNotExist() {
+			rc[proto][chainToString(table, chain)] = populated
+			return nil
+		}
+		return fmt.Errorf("failed to populate rules in chain %q for table %q: %v", chain, table, err)
+	}
+	for i := range rs {
+		rc[proto][strings.Join([]string{table, rs[i]}, " ")] = exists
+	}
+	// If there are rules on the chain, then the chain exists too.
+	if len(rs) > 0 {
+		rc[proto][chainToString(table, chain)] = exists
+	}
+	rc[proto][chainToString(table, chain)] |= populated
+	return nil
+}
+
+func (rc *ruleCache) populateRules(c Client, r Rule) error {
+	// Ensure a map for the proto exists.
+	if rc[r.Proto()] == nil {
+		rc[r.Proto()] = make(map[string]ruleCacheFlag)
+	}
+
+	if ch, ok := r.(*chain); ok {
+		return rc.populateTable(c, r.Proto(), ch.table)
+	}
+
+	ru := r.(*rule)
+	return rc.populateChain(c, r.Proto(), ru.table, ru.chain)
+}
+
+func (rc *ruleCache) exists(c Client, r Rule) (bool, error) {
+	// Exit early if the exact rule exists by name.
+	if rc[r.Proto()][r.String()]&exists != 0 {
+		return true, nil
+	}
+
+	// Otherwise, populate the respective rules.
+	if err := rc.populateRules(c, r); err != nil {
+		return false, err
+	}
+
+	return rc[r.Proto()][r.String()]&exists != 0, nil
+}

pkg/iptables/rulecache_test.go

@@ -0,0 +1,125 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"testing"
+)
+
+func TestRuleCache(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		rules []Rule
+		check []Rule
+		out   []bool
+		calls uint64
+	}{
+		{
+			name:  "empty",
+			rules: nil,
+			check: []Rule{rules[0]},
+			out:   []bool{false},
+			calls: 1,
+		},
+		{
+			name:  "single negative",
+			rules: []Rule{rules[1]},
+			check: []Rule{rules[0]},
+			out:   []bool{false},
+			calls: 1,
+		},
+		{
+			name:  "single positive",
+			rules: []Rule{rules[1]},
+			check: []Rule{rules[1]},
+			out:   []bool{true},
+			calls: 1,
+		},
+		{
+			name:  "single chain",
+			rules: []Rule{&chain{"nat", "KILO-NAT", ProtocolIPv4}},
+			check: []Rule{&chain{"nat", "KILO-NAT", ProtocolIPv4}},
+			out:   []bool{true},
+			calls: 1,
+		},
+		{
+			name:  "rule on chain means chain exists",
+			rules: []Rule{rules[0]},
+			check: []Rule{rules[0], &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{true, true},
+			calls: 1,
+		},
+		{
+			name:  "rule on chain does not mean table is fully populated",
+			rules: []Rule{rules[0], &chain{"filter", "INPUT", ProtocolIPv4}},
+			check: []Rule{rules[0], &chain{"filter", "OUTPUT", ProtocolIPv4}, &chain{"filter", "INPUT", ProtocolIPv4}},
+			out:   []bool{true, false, true},
+			calls: 2,
+		},
+		{
+			name:  "multiple rules on chain",
+			rules: []Rule{rules[0], rules[1]},
+			check: []Rule{rules[0], rules[1], &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{true, true, true},
+			calls: 1,
+		},
+		{
+			name:  "checking rule on chain does not mean chain exists",
+			rules: nil,
+			check: []Rule{rules[0], &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{false, false},
+			calls: 2,
+		},
+		{
+			name:  "multiple chains on same table",
+			rules: nil,
+			check: []Rule{&chain{"filter", "INPUT", ProtocolIPv4}, &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{false, false},
+			calls: 1,
+		},
+		{
+			name:  "multiple chains on different table",
+			rules: nil,
+			check: []Rule{&chain{"filter", "INPUT", ProtocolIPv4}, &chain{"nat", "POSTROUTING", ProtocolIPv4}},
+			out:   []bool{false, false},
+			calls: 2,
+		},
+	} {
+		controller := &Controller{}
+		client := &fakeClient{}
+		controller.v4 = client
+		controller.v6 = client
+		if err := controller.Set(tc.rules); err != nil {
+			t.Fatalf("test case %q: Set should not fail: %v", tc.name, err)
+		}
+		// Reset the client's calls so we can examine how many times
+		// the rule cache performs operations.
+		client.calls = 0
+		var rc ruleCache
+		for i := range tc.check {
+			ok, err := rc.exists(controller.client(tc.check[i].Proto()), tc.check[i])
+			if err != nil {
+				t.Fatalf("test case %q check %d: check should not fail: %v", tc.name, i, err)
+			}
+			if ok != tc.out[i] {
+				t.Errorf("test case %q check %d: expected %t, got %t", tc.name, i, tc.out[i], ok)
+			}
+		}
+		if client.calls != tc.calls {
+			t.Errorf("test case %q: expected client to be called %d times, got %d", tc.name, tc.calls, client.calls)
+		}
+	}
+
+}

pkg/k8s/apis/kilo/register.go

@@ -16,5 +16,5 @@ package kilo
 
 const (
 	// GroupName contains the API group name for Kilo API group.
-	GroupName = "kilo"
+	GroupName = "kilo.squat.ai"
 )

pkg/k8s/apis/kilo/v1alpha1/doc.go

@@ -15,5 +15,5 @@
 // +k8s:deepcopy-gen=package,register
 
 // Package v1alpha1 is the v1alpha1 version of the API.
-// +groupName=kilo
+// +groupName=kilo.squat.ai
 package v1alpha1

pkg/k8s/apis/kilo/v1alpha1/types.go

@@ -15,13 +15,16 @@
 package v1alpha1
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/validation"
 )
 
 const (
@@ -45,6 +48,7 @@ var PeerShortNames = []string{"peer"}
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +k8s:openapi-gen=true
+// +kubebuilder:resource:scope=Cluster
 
 // Peer is a WireGuard peer that should have access to the VPN.
 type Peer struct {
@@ -74,19 +78,30 @@ type PeerSpec struct {
 	PersistentKeepalive int `json:"persistentKeepalive,omitempty"`
 	// PresharedKey is the optional symmetric encryption key for the peer.
 	// +optional
-	PresharedKey string `json:"presharedKey"`
+	PresharedKey string `json:"presharedKey,omitempty"`
 	// PublicKey is the WireGuard public key for the peer.
 	PublicKey string `json:"publicKey"`
 }
 
-// PeerEndpoint represents a WireGuard enpoint, which is a ip:port tuple.
+// PeerEndpoint represents a WireGuard endpoint, which is an IP:port tuple.
 type PeerEndpoint struct {
-	// IP must be a valid IP address.
+	// DNSOrIP is a DNS name or an IP address.
-	IP string `json:"ip"`
+	DNSOrIP `json:"dnsOrIP"`
 	// Port must be a valid port number.
 	Port uint32 `json:"port"`
 }
 
+// DNSOrIP represents either a DNS name or an IP address.
+// When both are given, the IP address, as it is more specific, override the DNS name.
+type DNSOrIP struct {
+	// DNS must be a valid RFC 1123 subdomain.
+	// +optional
+	DNS string `json:"dns,omitempty"`
+	// IP must be a valid IP address.
+	// +optional
+	IP string `json:"ip,omitempty"`
+}
+
 // PeerName is the peer resource's FQDN.
 var PeerName = PeerPlural + "." + GroupName
 
@@ -121,24 +136,35 @@ func (p *Peer) Copy() *Peer {
 func (p *Peer) Validate() error {
 	for _, ip := range p.Spec.AllowedIPs {
 		if _, n, err := net.ParseCIDR(ip); err != nil {
-			return fmt.Errorf("failed to parse %q as a valid IP address: %v", ip, err)
+			return fmt.Errorf("failed to parse %q as a valid IP address: %w", ip, err)
 		} else if n == nil {
 			return fmt.Errorf("got invalid IP address for %q", ip)
 		}
 	}
 	if p.Spec.Endpoint != nil {
-		if net.ParseIP(p.Spec.Endpoint.IP) == nil {
+		if p.Spec.Endpoint.IP == "" && p.Spec.Endpoint.DNS == "" {
+			return errors.New("either an endpoint DNS name IP address must be given")
+		}
+		if p.Spec.Endpoint.DNS != "" {
+			if errs := validation.IsDNS1123Subdomain(p.Spec.Endpoint.DNS); len(errs) != 0 {
+				return errors.New(strings.Join(errs, "; "))
+			}
+		}
+		if p.Spec.Endpoint.IP != "" && net.ParseIP(p.Spec.Endpoint.IP) == nil {
 			return fmt.Errorf("failed to parse %q as a valid IP address", p.Spec.Endpoint.IP)
 		}
-		if p.Spec.Endpoint.Port == 0 {
+		if 1 > p.Spec.Endpoint.Port || p.Spec.Endpoint.Port > 65535 {
 			return fmt.Errorf("port must be a valid UDP port number, got %d", p.Spec.Endpoint.Port)
 		}
 	}
 	if p.Spec.PersistentKeepalive < 0 {
 		return fmt.Errorf("persistent keepalive must be greater than or equal to zero; got %q", p.Spec.PersistentKeepalive)
 	}
-	if len(p.Spec.PublicKey) == 0 {
+	if b, err := base64.StdEncoding.DecodeString(p.Spec.PublicKey); err != nil {
-		return errors.New("public keys cannot be empty")
+		return fmt.Errorf("WireGuard public key is not base64 encoded: %w", err)
+		// Since WireGuard is using Curve25519 for the key exchange, the key length of 256 bits should not change in the near future.
+	} else if len(b) != 32 {
+		return errors.New("WireGuard public key has invalid length")
 	}
 	return nil
 }
@@ -148,6 +174,9 @@ func (p *Peer) Validate() error {
 // PeerList is a list of peers.
 type PeerList struct {
 	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	// +optional
 	metav1.ListMeta `json:"metadata,omitempty"`
 	// List of peers.
 	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md

pkg/k8s/apis/kilo/v1alpha1/zz_generated.deepcopy.go

@@ -1,6 +1,6 @@
 // +build !ignore_autogenerated
 
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,6 +22,22 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DNSOrIP) DeepCopyInto(out *DNSOrIP) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSOrIP.
+func (in *DNSOrIP) DeepCopy() *DNSOrIP {
+	if in == nil {
+		return nil
+	}
+	out := new(DNSOrIP)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Peer) DeepCopyInto(out *Peer) {
 	*out = *in
@@ -52,6 +68,7 @@ func (in *Peer) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PeerEndpoint) DeepCopyInto(out *PeerEndpoint) {
 	*out = *in
+	out.DNSOrIP = in.DNSOrIP
 	return
 }
 
@@ -69,7 +86,7 @@ func (in *PeerEndpoint) DeepCopy() *PeerEndpoint {
 func (in *PeerList) DeepCopyInto(out *PeerList) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
-	out.ListMeta = in.ListMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
 	if in.Items != nil {
 		in, out := &in.Items, &out.Items
 		*out = make([]Peer, len(*in))

pkg/k8s/backend.go

@@ -15,6 +15,7 @@
 package k8s
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -24,11 +25,9 @@ import (
 	"strings"
 	"time"
 
-	crdutils "github.com/ant31/crd-validation/pkg"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
@@ -59,8 +58,11 @@ const (
 	locationAnnotationKey        = "kilo.squat.ai/location"
 	persistentKeepaliveKey       = "kilo.squat.ai/persistent-keepalive"
 	wireGuardIPAnnotationKey     = "kilo.squat.ai/wireguard-ip"
-
+	discoveredEndpointsKey       = "kilo.squat.ai/discovered-endpoints"
-	regionLabelKey  = "topology.kubernetes.io/region"
+	allowedLocationIPsKey        = "kilo.squat.ai/allowed-location-ips"
+	granularityKey               = "kilo.squat.ai/granularity"
+	// RegionLabelKey is the key for the well-known Kubernetes topology region label.
+	RegionLabelKey  = "topology.kubernetes.io/region"
 	jsonPatchSlash  = "~1"
 	jsonRemovePatch = `{"op": "remove", "path": "%s"}`
 )
@@ -85,6 +87,7 @@ type nodeBackend struct {
 	events        chan *mesh.NodeEvent
 	informer      cache.SharedIndexInformer
 	lister        v1listers.NodeLister
+	topologyLabel string
 }
 
 type peerBackend struct {
@@ -96,7 +99,7 @@ type peerBackend struct {
 }
 
 // New creates a new instance of a mesh.Backend.
-func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Interface) mesh.Backend {
+func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Interface, topologyLabel string) mesh.Backend {
 	ni := v1informers.NewNodeInformer(c, 5*time.Minute, nil)
 	pi := v1alpha1informers.NewPeerInformer(kc, 5*time.Minute, nil)
 
@@ -106,6 +109,7 @@ func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Inter
 			events:        make(chan *mesh.NodeEvent),
 			informer:      ni,
 			lister:        v1listers.NewNodeLister(ni.GetIndexer()),
+			topologyLabel: topologyLabel,
 		},
 		&peerBackend{
 			client:           kc,
@@ -125,8 +129,10 @@ func (nb *nodeBackend) CleanUp(name string) error {
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(keyAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(lastSeenAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(wireGuardIPAnnotationKey, "/", jsonPatchSlash, 1))),
+		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(discoveredEndpointsKey, "/", jsonPatchSlash, 1))),
+		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(granularityKey, "/", jsonPatchSlash, 1))),
 	}, ",") + "]")
-	if _, err := nb.client.CoreV1().Nodes().Patch(name, types.JSONPatchType, patch); err != nil {
+	if _, err := nb.client.CoreV1().Nodes().Patch(context.TODO(), name, types.JSONPatchType, patch, metav1.PatchOptions{}); err != nil {
 		return fmt.Errorf("failed to patch node: %v", err)
 	}
 	return nil
@@ -138,7 +144,7 @@ func (nb *nodeBackend) Get(name string) (*mesh.Node, error) {
 	if err != nil {
 		return nil, err
 	}
-	return translateNode(n), nil
+	return translateNode(n, nb.topologyLabel), nil
 }
 
 // Init initializes the backend; for this backend that means
@@ -158,7 +164,7 @@ func (nb *nodeBackend) Init(stop <-chan struct{}) error {
 					// Failed to decode Node; ignoring...
 					return
 				}
-				nb.events <- &mesh.NodeEvent{Type: mesh.AddEvent, Node: translateNode(n)}
+				nb.events <- &mesh.NodeEvent{Type: mesh.AddEvent, Node: translateNode(n, nb.topologyLabel)}
 			},
 			UpdateFunc: func(old, obj interface{}) {
 				n, ok := obj.(*v1.Node)
@@ -171,7 +177,7 @@ func (nb *nodeBackend) Init(stop <-chan struct{}) error {
 					// Failed to decode Node; ignoring...
 					return
 				}
-				nb.events <- &mesh.NodeEvent{Type: mesh.UpdateEvent, Node: translateNode(n), Old: translateNode(o)}
+				nb.events <- &mesh.NodeEvent{Type: mesh.UpdateEvent, Node: translateNode(n, nb.topologyLabel), Old: translateNode(o, nb.topologyLabel)}
 			},
 			DeleteFunc: func(obj interface{}) {
 				n, ok := obj.(*v1.Node)
@@ -179,7 +185,7 @@ func (nb *nodeBackend) Init(stop <-chan struct{}) error {
 					// Failed to decode Node; ignoring...
 					return
 				}
-				nb.events <- &mesh.NodeEvent{Type: mesh.DeleteEvent, Node: translateNode(n)}
+				nb.events <- &mesh.NodeEvent{Type: mesh.DeleteEvent, Node: translateNode(n, nb.topologyLabel)}
 			},
 		},
 	)
@@ -194,7 +200,7 @@ func (nb *nodeBackend) List() ([]*mesh.Node, error) {
 	}
 	nodes := make([]*mesh.Node, len(ns))
 	for i := range ns {
-		nodes[i] = translateNode(ns[i])
+		nodes[i] = translateNode(ns[i], nb.topologyLabel)
 	}
 	return nodes, nil
 }
@@ -207,7 +213,11 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
 	}
 	n := old.DeepCopy()
 	n.ObjectMeta.Annotations[endpointAnnotationKey] = node.Endpoint.String()
+	if node.InternalIP == nil {
+		n.ObjectMeta.Annotations[internalIPAnnotationKey] = ""
+	} else {
 		n.ObjectMeta.Annotations[internalIPAnnotationKey] = node.InternalIP.String()
+	}
 	n.ObjectMeta.Annotations[keyAnnotationKey] = string(node.Key)
 	n.ObjectMeta.Annotations[lastSeenAnnotationKey] = strconv.FormatInt(node.LastSeen, 10)
 	if node.WireGuardIP == nil {
@@ -215,6 +225,16 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
 	} else {
 		n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = node.WireGuardIP.String()
 	}
+	if node.DiscoveredEndpoints == nil {
+		n.ObjectMeta.Annotations[discoveredEndpointsKey] = ""
+	} else {
+		discoveredEndpoints, err := json.Marshal(node.DiscoveredEndpoints)
+		if err != nil {
+			return err
+		}
+		n.ObjectMeta.Annotations[discoveredEndpointsKey] = string(discoveredEndpoints)
+	}
+	n.ObjectMeta.Annotations[granularityKey] = string(node.Granularity)
 	oldData, err := json.Marshal(old)
 	if err != nil {
 		return err
@@ -227,7 +247,7 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
 	if err != nil {
 		return fmt.Errorf("failed to create patch for node %q: %v", n.Name, err)
 	}
-	if _, err = nb.client.CoreV1().Nodes().Patch(name, types.StrategicMergePatchType, patch); err != nil {
+	if _, err = nb.client.CoreV1().Nodes().Patch(context.TODO(), name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
 		return fmt.Errorf("failed to patch node: %v", err)
 	}
 	return nil
@@ -239,7 +259,7 @@ func (nb *nodeBackend) Watch() <-chan *mesh.NodeEvent {
 }
 
 // translateNode translates a Kubernetes Node to a mesh.Node.
-func translateNode(node *v1.Node) *mesh.Node {
+func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 	if node == nil {
 		return nil
 	}
@@ -253,7 +273,7 @@ func translateNode(node *v1.Node) *mesh.Node {
 	// Allow the region to be overridden by an explicit location.
 	location, ok := node.ObjectMeta.Annotations[locationAnnotationKey]
 	if !ok {
-		location = node.ObjectMeta.Labels[regionLabelKey]
+		location = node.ObjectMeta.Labels[topologyLabel]
 	}
 	// Allow the endpoint to be overridden.
 	endpoint := parseEndpoint(node.ObjectMeta.Annotations[forceEndpointAnnotationKey])
@@ -265,6 +285,12 @@ func translateNode(node *v1.Node) *mesh.Node {
 	if internalIP == nil {
 		internalIP = normalizeIP(node.ObjectMeta.Annotations[internalIPAnnotationKey])
 	}
+	// Set the ForceInternalIP flag, if force-internal-ip annotation was set to "".
+	noInternalIP := false
+	if s, ok := node.ObjectMeta.Annotations[forceInternalIPAnnotationKey]; ok && (s == "" || s == "-") {
+		noInternalIP = true
+		internalIP = nil
+	}
 	// Set Wireguard PersistentKeepalive setting for the node.
 	var persistentKeepalive int64
 	if keepAlive, ok := node.ObjectMeta.Annotations[persistentKeepaliveKey]; !ok {
@@ -282,12 +308,42 @@ func translateNode(node *v1.Node) *mesh.Node {
 			lastSeen = 0
 		}
 	}
+	var discoveredEndpoints map[string]*wireguard.Endpoint
+	if de, ok := node.ObjectMeta.Annotations[discoveredEndpointsKey]; ok {
+		err := json.Unmarshal([]byte(de), &discoveredEndpoints)
+		if err != nil {
+			discoveredEndpoints = nil
+		}
+	}
+	// Set allowed IPs for a location.
+	var allowedLocationIPs []*net.IPNet
+	if str, ok := node.ObjectMeta.Annotations[allowedLocationIPsKey]; ok {
+		for _, ip := range strings.Split(str, ",") {
+			if ipnet := normalizeIP(ip); ipnet != nil {
+				allowedLocationIPs = append(allowedLocationIPs, ipnet)
+			}
+		}
+	}
+	var meshGranularity mesh.Granularity
+	if gr, ok := node.ObjectMeta.Annotations[granularityKey]; ok {
+		meshGranularity = mesh.Granularity(gr)
+		switch meshGranularity {
+		case mesh.LogicalGranularity:
+		case mesh.FullGranularity:
+		default:
+			meshGranularity = ""
+		}
+	}
+
 	return &mesh.Node{
 		// Endpoint and InternalIP should only ever fail to parse if the
 		// remote node's agent has not yet set its IP address;
 		// in this case the IP will be nil and
 		// the mesh can wait for the node to be updated.
+		// It is valid for the InternalIP to be nil,
+		// if the given node only has public IP addresses.
 		Endpoint:            endpoint,
+		NoInternalIP:        noInternalIP,
 		InternalIP:          internalIP,
 		Key:                 []byte(node.ObjectMeta.Annotations[keyAnnotationKey]),
 		LastSeen:            lastSeen,
@@ -300,6 +356,9 @@ func translateNode(node *v1.Node) *mesh.Node {
 		// the node's agent has not yet reconciled. In either case, the IP
 		// will parse as nil.
 		WireGuardIP:         normalizeIP(node.ObjectMeta.Annotations[wireGuardIPAnnotationKey]),
+		DiscoveredEndpoints: discoveredEndpoints,
+		AllowedLocationIPs:  allowedLocationIPs,
+		Granularity:         meshGranularity,
 	}
 }
 
@@ -325,9 +384,12 @@ func translatePeer(peer *v1alpha1.Peer) *mesh.Peer {
 		} else {
 			ip = ip.To16()
 		}
-		if peer.Spec.Endpoint.Port > 0 && ip != nil {
+		if peer.Spec.Endpoint.Port > 0 && (ip != nil || peer.Spec.Endpoint.DNS != "") {
 			endpoint = &wireguard.Endpoint{
-				DNSOrIP: wireguard.DNSOrIP{IP: ip},
+				DNSOrIP: wireguard.DNSOrIP{
+					DNS: peer.Spec.Endpoint.DNS,
+					IP:  ip,
+				},
 				Port: peer.Spec.Endpoint.Port,
 			}
 		}
@@ -373,24 +435,9 @@ func (pb *peerBackend) Get(name string) (*mesh.Peer, error) {
 // Init initializes the backend; for this backend that means
 // syncing the informer cache.
 func (pb *peerBackend) Init(stop <-chan struct{}) error {
-	// Register CRD.
+	// Check the presents of the CRD peers.kilo.squat.ai.
-	crd := crdutils.NewCustomResourceDefinition(crdutils.Config{
+	if _, err := pb.extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), strings.Join([]string{v1alpha1.PeerPlural, v1alpha1.GroupName}, "."), metav1.GetOptions{}); err != nil {
-		SpecDefinitionName:    "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1.Peer",
+		return fmt.Errorf("CRD is not present: %v", err)
-		EnableValidation:      true,
-		ResourceScope:         string(v1beta1.ClusterScoped),
-		Group:                 v1alpha1.GroupName,
-		Kind:                  v1alpha1.PeerKind,
-		Version:               v1alpha1.SchemeGroupVersion.Version,
-		Plural:                v1alpha1.PeerPlural,
-		ShortNames:            v1alpha1.PeerShortNames,
-		GetOpenAPIDefinitions: v1alpha1.GetOpenAPIDefinitions,
-	})
-	crd.Spec.Subresources.Scale = nil
-	crd.Spec.Subresources.Status = nil
-
-	_, err := pb.extensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
-	if err != nil && !apierrors.IsAlreadyExists(err) {
-		return fmt.Errorf("failed to create CRD: %v", err)
 	}
 
 	go pb.informer.Run(stop)
@@ -464,15 +511,22 @@ func (pb *peerBackend) Set(name string, peer *mesh.Peer) error {
 		p.Spec.AllowedIPs[i] = peer.AllowedIPs[i].String()
 	}
 	if peer.Endpoint != nil {
+		var ip string
+		if peer.Endpoint.IP != nil {
+			ip = peer.Endpoint.IP.String()
+		}
 		p.Spec.Endpoint = &v1alpha1.PeerEndpoint{
-			IP:   peer.Endpoint.IP.String(),
+			DNSOrIP: v1alpha1.DNSOrIP{
+				IP:  ip,
+				DNS: peer.Endpoint.DNS,
+			},
 			Port: peer.Endpoint.Port,
 		}
 	}
 	p.Spec.PersistentKeepalive = peer.PersistentKeepalive
 	p.Spec.PresharedKey = string(peer.PresharedKey)
 	p.Spec.PublicKey = string(peer.PublicKey)
-	if _, err = pb.client.KiloV1alpha1().Peers().Update(p); err != nil {
+	if _, err = pb.client.KiloV1alpha1().Peers().Update(context.TODO(), p, metav1.UpdateOptions{}); err != nil {
 		return fmt.Errorf("failed to update peer: %v", err)
 	}
 	return nil

pkg/k8s/backend_test.go

@@ -83,7 +83,7 @@ func TestTranslateNode(t *testing.T) {
 		{
 			name: "region",
 			labels: map[string]string{
-				regionLabelKey: "a",
+				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
 				Location: "a",
@@ -95,7 +95,7 @@ func TestTranslateNode(t *testing.T) {
 				locationAnnotationKey: "b",
 			},
 			labels: map[string]string{
-				regionLabelKey: "a",
+				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
 				Location: "b",
@@ -138,6 +138,7 @@ func TestTranslateNode(t *testing.T) {
 			},
 			out: &mesh.Node{
 				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.1"), Mask: net.CIDRMask(24, 32)},
+				NoInternalIP: false,
 			},
 		},
 		{
@@ -148,6 +149,7 @@ func TestTranslateNode(t *testing.T) {
 			},
 			out: &mesh.Node{
 				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(24, 32)},
+				NoInternalIP: false,
 			},
 		},
 		{
@@ -172,10 +174,11 @@ func TestTranslateNode(t *testing.T) {
 				wireGuardIPAnnotationKey:     "10.4.0.1/16",
 			},
 			labels: map[string]string{
-				regionLabelKey: "a",
+				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
 				Endpoint:            &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.2")}, Port: 51821},
+				NoInternalIP:        false,
 				InternalIP:          &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(32, 32)},
 				Key:                 []byte("foo"),
 				LastSeen:            1000000000,
@@ -187,12 +190,68 @@ func TestTranslateNode(t *testing.T) {
 			},
 			subnet: "10.2.1.0/24",
 		},
+		{
+			name: "no InternalIP",
+			annotations: map[string]string{
+				endpointAnnotationKey:    "10.0.0.1:51820",
+				internalIPAnnotationKey:  "",
+				keyAnnotationKey:         "foo",
+				lastSeenAnnotationKey:    "1000000000",
+				locationAnnotationKey:    "b",
+				persistentKeepaliveKey:   "25",
+				wireGuardIPAnnotationKey: "10.4.0.1/16",
+			},
+			labels: map[string]string{
+				RegionLabelKey: "a",
+			},
+			out: &mesh.Node{
+				Endpoint:            &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: 51820},
+				InternalIP:          nil,
+				Key:                 []byte("foo"),
+				LastSeen:            1000000000,
+				Leader:              false,
+				Location:            "b",
+				PersistentKeepalive: 25,
+				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
+			},
+			subnet: "10.2.1.0/24",
+		},
+		{
+			name: "Force no internal IP",
+			annotations: map[string]string{
+				endpointAnnotationKey:        "10.0.0.1:51820",
+				internalIPAnnotationKey:      "10.1.0.1/32",
+				forceInternalIPAnnotationKey: "",
+				keyAnnotationKey:             "foo",
+				lastSeenAnnotationKey:        "1000000000",
+				locationAnnotationKey:        "b",
+				persistentKeepaliveKey:       "25",
+				wireGuardIPAnnotationKey:     "10.4.0.1/16",
+			},
+			labels: map[string]string{
+				RegionLabelKey: "a",
+			},
+			out: &mesh.Node{
+				Endpoint:            &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: 51820},
+				NoInternalIP:        true,
+				InternalIP:          nil,
+				Key:                 []byte("foo"),
+				LastSeen:            1000000000,
+				Leader:              false,
+				Location:            "b",
+				PersistentKeepalive: 25,
+				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
+			},
+			subnet: "10.2.1.0/24",
+		},
 	} {
 		n := &v1.Node{}
 		n.ObjectMeta.Annotations = tc.annotations
 		n.ObjectMeta.Labels = tc.labels
 		n.Spec.PodCIDR = tc.subnet
-		node := translateNode(n)
+		node := translateNode(n, RegionLabelKey)
 		if diff := pretty.Compare(node, tc.out); diff != "" {
 			t.Errorf("test case %q: got diff: %v", tc.name, diff)
 		}
@@ -240,17 +299,30 @@ func TestTranslatePeer(t *testing.T) {
 			name: "invalid endpoint ip",
 			spec: v1alpha1.PeerSpec{
 				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
 						IP: "foo",
+					},
 					Port: mesh.DefaultKiloPort,
 				},
 			},
 			out: &mesh.Peer{},
 		},
 		{
-			name: "valid endpoint",
+			name: "only endpoint port",
 			spec: v1alpha1.PeerSpec{
 				Endpoint: &v1alpha1.PeerEndpoint{
+					Port: mesh.DefaultKiloPort,
+				},
+			},
+			out: &mesh.Peer{},
+		},
+		{
+			name: "valid endpoint ip",
+			spec: v1alpha1.PeerSpec{
+				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
 						IP: "10.0.0.1",
+					},
 					Port: mesh.DefaultKiloPort,
 				},
 			},
@@ -263,6 +335,25 @@ func TestTranslatePeer(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "valid endpoint DNS",
+			spec: v1alpha1.PeerSpec{
+				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
+						DNS: "example.com",
+					},
+					Port: mesh.DefaultKiloPort,
+				},
+			},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					Endpoint: &wireguard.Endpoint{
+						DNSOrIP: wireguard.DNSOrIP{DNS: "example.com"},
+						Port:    mesh.DefaultKiloPort,
+					},
+				},
+			},
+		},
 		{
 			name: "empty key",
 			spec: v1alpha1.PeerSpec{

pkg/k8s/clientset/versioned/clientset.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 package versioned
 
 import (
+	"fmt"
+
 	kilov1alpha1 "github.com/squat/kilo/pkg/k8s/clientset/versioned/typed/kilo/v1alpha1"
 	discovery "k8s.io/client-go/discovery"
 	rest "k8s.io/client-go/rest"
@@ -49,9 +51,14 @@ func (c *Clientset) Discovery() discovery.DiscoveryInterface {
 }
 
 // NewForConfig creates a new Clientset for the given config.
+// If config's RateLimiter is not set and QPS and Burst are acceptable,
+// NewForConfig will generate a rate-limiter in configShallowCopy.
 func NewForConfig(c *rest.Config) (*Clientset, error) {
 	configShallowCopy := *c
 	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
+		if configShallowCopy.Burst <= 0 {
+			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
+		}
 		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
 	}
 	var cs Clientset

pkg/k8s/clientset/versioned/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/fake/clientset_generated.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -39,7 +39,7 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset {
 		}
 	}
 
-	cs := &Clientset{}
+	cs := &Clientset{tracker: o}
 	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
 	cs.AddReactor("*", "*", testing.ObjectReaction(o))
 	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
@@ -61,12 +61,17 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset {
 type Clientset struct {
 	testing.Fake
 	discovery *fakediscovery.FakeDiscovery
+	tracker   testing.ObjectTracker
 }
 
 func (c *Clientset) Discovery() discovery.DiscoveryInterface {
 	return c.discovery
 }
 
+func (c *Clientset) Tracker() testing.ObjectTracker {
+	return c.tracker
+}
+
 var _ clientset.Interface = &Clientset{}
 
 // KiloV1alpha1 retrieves the KiloV1alpha1Client

pkg/k8s/clientset/versioned/fake/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/fake/register.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ import (
 
 var scheme = runtime.NewScheme()
 var codecs = serializer.NewCodecFactory(scheme)
-var parameterCodec = runtime.NewParameterCodec(scheme)
+
 var localSchemeBuilder = runtime.SchemeBuilder{
 	kilov1alpha1.AddToScheme,
 }

pkg/k8s/clientset/versioned/scheme/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/scheme/register.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/fake_kilo_client.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/fake_peer.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 package fake
 
 import (
+	"context"
+
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	labels "k8s.io/apimachinery/pkg/labels"
@@ -31,12 +33,12 @@ type FakePeers struct {
 	Fake *FakeKiloV1alpha1
 }
 
-var peersResource = schema.GroupVersionResource{Group: "kilo", Version: "v1alpha1", Resource: "peers"}
+var peersResource = schema.GroupVersionResource{Group: "kilo.squat.ai", Version: "v1alpha1", Resource: "peers"}
 
-var peersKind = schema.GroupVersionKind{Group: "kilo", Version: "v1alpha1", Kind: "Peer"}
+var peersKind = schema.GroupVersionKind{Group: "kilo.squat.ai", Version: "v1alpha1", Kind: "Peer"}
 
 // Get takes name of the peer, and returns the corresponding peer object, and an error if there is any.
-func (c *FakePeers) Get(name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootGetAction(peersResource, name), &v1alpha1.Peer{})
 	if obj == nil {
@@ -46,7 +48,7 @@ func (c *FakePeers) Get(name string, options v1.GetOptions) (result *v1alpha1.Pe
 }
 
 // List takes label and field selectors, and returns the list of Peers that match those selectors.
-func (c *FakePeers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
+func (c *FakePeers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootListAction(peersResource, peersKind, opts), &v1alpha1.PeerList{})
 	if obj == nil {
@@ -67,13 +69,13 @@ func (c *FakePeers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err er
 }
 
 // Watch returns a watch.Interface that watches the requested peers.
-func (c *FakePeers) Watch(opts v1.ListOptions) (watch.Interface, error) {
+func (c *FakePeers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
 	return c.Fake.
 		InvokesWatch(testing.NewRootWatchAction(peersResource, opts))
 }
 
 // Create takes the representation of a peer and creates it.  Returns the server's representation of the peer, and an error, if there is any.
-func (c *FakePeers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootCreateAction(peersResource, peer), &v1alpha1.Peer{})
 	if obj == nil {
@@ -83,7 +85,7 @@ func (c *FakePeers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err erro
 }
 
 // Update takes the representation of a peer and updates it. Returns the server's representation of the peer, and an error, if there is any.
-func (c *FakePeers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootUpdateAction(peersResource, peer), &v1alpha1.Peer{})
 	if obj == nil {
@@ -93,22 +95,22 @@ func (c *FakePeers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err erro
 }
 
 // Delete takes name of the peer and deletes it. Returns an error if one occurs.
-func (c *FakePeers) Delete(name string, options *v1.DeleteOptions) error {
+func (c *FakePeers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	_, err := c.Fake.
 		Invokes(testing.NewRootDeleteAction(peersResource, name), &v1alpha1.Peer{})
 	return err
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *FakePeers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+func (c *FakePeers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
-	action := testing.NewRootDeleteCollectionAction(peersResource, listOptions)
+	action := testing.NewRootDeleteCollectionAction(peersResource, listOpts)
 
 	_, err := c.Fake.Invokes(action, &v1alpha1.PeerList{})
 	return err
 }
 
 // Patch applies the patch and returns the patched peer.
-func (c *FakePeers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootPatchSubresourceAction(peersResource, name, pt, data, subresources...), &v1alpha1.Peer{})
 	if obj == nil {

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/generated_expansion.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/kilo_client.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ package v1alpha1
 import (
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
 	"github.com/squat/kilo/pkg/k8s/clientset/versioned/scheme"
-	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
 	rest "k8s.io/client-go/rest"
 )
 
@@ -28,7 +27,7 @@ type KiloV1alpha1Interface interface {
 	PeersGetter
 }
 
-// KiloV1alpha1Client is used to interact with features provided by the kilo group.
+// KiloV1alpha1Client is used to interact with features provided by the kilo.squat.ai group.
 type KiloV1alpha1Client struct {
 	restClient rest.Interface
 }
@@ -69,7 +68,7 @@ func setConfigDefaults(config *rest.Config) error {
 	gv := v1alpha1.SchemeGroupVersion
 	config.GroupVersion = &gv
 	config.APIPath = "/apis"
-	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
+	config.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
 
 	if config.UserAgent == "" {
 		config.UserAgent = rest.DefaultKubernetesUserAgent()

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package v1alpha1
 
 import (
+	"context"
 	"time"
 
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
@@ -35,14 +36,14 @@ type PeersGetter interface {
 
 // PeerInterface has methods to work with Peer resources.
 type PeerInterface interface {
-	Create(*v1alpha1.Peer) (*v1alpha1.Peer, error)
+	Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (*v1alpha1.Peer, error)
-	Update(*v1alpha1.Peer) (*v1alpha1.Peer, error)
+	Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (*v1alpha1.Peer, error)
-	Delete(name string, options *v1.DeleteOptions) error
+	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
-	DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
+	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
-	Get(name string, options v1.GetOptions) (*v1alpha1.Peer, error)
+	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.Peer, error)
-	List(opts v1.ListOptions) (*v1alpha1.PeerList, error)
+	List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.PeerList, error)
-	Watch(opts v1.ListOptions) (watch.Interface, error)
+	Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error)
-	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error)
 	PeerExpansion
 }
 
@@ -59,19 +60,19 @@ func newPeers(c *KiloV1alpha1Client) *peers {
 }
 
 // Get takes name of the peer, and returns the corresponding peer object, and an error if there is any.
-func (c *peers) Get(name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
+func (c *peers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Get().
 		Resource("peers").
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // List takes label and field selectors, and returns the list of Peers that match those selectors.
-func (c *peers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
+func (c *peers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -81,13 +82,13 @@ func (c *peers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error)
 		Resource("peers").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Watch returns a watch.Interface that watches the requested peers.
-func (c *peers) Watch(opts v1.ListOptions) (watch.Interface, error) {
+func (c *peers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -97,66 +98,69 @@ func (c *peers) Watch(opts v1.ListOptions) (watch.Interface, error) {
 		Resource("peers").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Watch()
+		Watch(ctx)
 }
 
 // Create takes the representation of a peer and creates it.  Returns the server's representation of the peer, and an error, if there is any.
-func (c *peers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *peers) Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Post().
 		Resource("peers").
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(peer).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Update takes the representation of a peer and updates it. Returns the server's representation of the peer, and an error, if there is any.
-func (c *peers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *peers) Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Put().
 		Resource("peers").
 		Name(peer.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(peer).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Delete takes name of the peer and deletes it. Returns an error if one occurs.
-func (c *peers) Delete(name string, options *v1.DeleteOptions) error {
+func (c *peers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	return c.client.Delete().
 		Resource("peers").
 		Name(name).
-		Body(options).
+		Body(&opts).
-		Do().
+		Do(ctx).
 		Error()
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *peers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+func (c *peers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
 	var timeout time.Duration
-	if listOptions.TimeoutSeconds != nil {
+	if listOpts.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
 	}
 	return c.client.Delete().
 		Resource("peers").
-		VersionedParams(&listOptions, scheme.ParameterCodec).
+		VersionedParams(&listOpts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Body(options).
+		Body(&opts).
-		Do().
+		Do(ctx).
 		Error()
 }
 
 // Patch applies the patch and returns the patched peer.
-func (c *peers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error) {
+func (c *peers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Patch(pt).
 		Resource("peers").
-		SubResource(subresources...).
 		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(data).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }

pkg/k8s/informers/factory.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/generic.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ func (f *genericInformer) Lister() cache.GenericLister {
 // TODO extend this to unknown resources with a client pool
 func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {
 	switch resource {
-	// Group=kilo, Version=v1alpha1
+	// Group=kilo.squat.ai, Version=v1alpha1
 	case v1alpha1.SchemeGroupVersion.WithResource("peers"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Kilo().V1alpha1().Peers().Informer()}, nil
 

pkg/k8s/informers/internalinterfaces/factory_interfaces.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/kilo/interface.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/kilo/v1alpha1/interface.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/kilo/v1alpha1/peer.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package v1alpha1
 
 import (
+	"context"
 	time "time"
 
 	kilov1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
@@ -58,13 +59,13 @@ func NewFilteredPeerInformer(client versioned.Interface, resyncPeriod time.Durat
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.KiloV1alpha1().Peers().List(options)
+				return client.KiloV1alpha1().Peers().List(context.TODO(), options)
 			},
 			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.KiloV1alpha1().Peers().Watch(options)
+				return client.KiloV1alpha1().Peers().Watch(context.TODO(), options)
 			},
 		},
 		&kilov1alpha1.Peer{},

pkg/k8s/listers/kilo/v1alpha1/expansion_generated.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/listers/kilo/v1alpha1/peer.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -24,10 +24,13 @@ import (
 )
 
 // PeerLister helps list Peers.
+// All objects returned here must be treated as read-only.
 type PeerLister interface {
 	// List lists all Peers in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1alpha1.Peer, err error)
 	// Get retrieves the Peer from the index for a given name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1alpha1.Peer, error)
 	PeerListerExpansion
 }

pkg/mesh/backend.go

@@ -0,0 +1,159 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mesh
+
+import (
+	"net"
+	"time"
+
+	"github.com/squat/kilo/pkg/wireguard"
+)
+
+const (
+	// checkInPeriod is how often nodes should check-in.
+	checkInPeriod = 30 * time.Second
+	// DefaultKiloInterface is the default interface created and used by Kilo.
+	DefaultKiloInterface = "kilo0"
+	// DefaultKiloPort is the default UDP port Kilo uses.
+	DefaultKiloPort = 51820
+	// DefaultCNIPath is the default path to the CNI config file.
+	DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist"
+)
+
+// DefaultKiloSubnet is the default CIDR for Kilo.
+var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}}
+
+// Granularity represents the abstraction level at which the network
+// should be meshed.
+type Granularity string
+
+const (
+	// LogicalGranularity indicates that the network should create
+	// a mesh between logical locations, e.g. data-centers, but not between
+	// all nodes within a single location.
+	LogicalGranularity Granularity = "location"
+	// FullGranularity indicates that the network should create
+	// a mesh between every node.
+	FullGranularity Granularity = "full"
+	// AutoGranularity can be used with kgctl to obtain
+	// the granularity automatically.
+	AutoGranularity Granularity = "auto"
+)
+
+// Node represents a node in the network.
+type Node struct {
+	Endpoint     *wireguard.Endpoint
+	Key          []byte
+	NoInternalIP bool
+	InternalIP   *net.IPNet
+	// LastSeen is a Unix time for the last time
+	// the node confirmed it was live.
+	LastSeen int64
+	// Leader is a suggestion to Kilo that
+	// the node wants to lead its segment.
+	Leader              bool
+	Location            string
+	Name                string
+	PersistentKeepalive int
+	Subnet              *net.IPNet
+	WireGuardIP         *net.IPNet
+	DiscoveredEndpoints map[string]*wireguard.Endpoint
+	AllowedLocationIPs  []*net.IPNet
+	Granularity         Granularity
+}
+
+// Ready indicates whether or not the node is ready.
+func (n *Node) Ready() bool {
+	// Nodes that are not leaders will not have WireGuardIPs, so it is not required.
+	return n != nil && n.Endpoint != nil && !(n.Endpoint.IP == nil && n.Endpoint.DNS == "") && n.Endpoint.Port != 0 && n.Key != nil && n.Subnet != nil && time.Now().Unix()-n.LastSeen < int64(checkInPeriod)*2/int64(time.Second)
+}
+
+// Peer represents a peer in the network.
+type Peer struct {
+	wireguard.Peer
+	Name string
+}
+
+// Ready indicates whether or not the peer is ready.
+// Peers can have empty endpoints because they may not have an
+// IP, for example if they are behind a NAT, and thus
+// will not declare their endpoint and instead allow it to be
+// discovered.
+func (p *Peer) Ready() bool {
+	return p != nil && p.AllowedIPs != nil && len(p.AllowedIPs) != 0 && p.PublicKey != nil
+}
+
+// EventType describes what kind of an action an event represents.
+type EventType string
+
+const (
+	// AddEvent represents an action where an item was added.
+	AddEvent EventType = "add"
+	// DeleteEvent represents an action where an item was removed.
+	DeleteEvent EventType = "delete"
+	// UpdateEvent represents an action where an item was updated.
+	UpdateEvent EventType = "update"
+)
+
+// NodeEvent represents an event concerning a node in the cluster.
+type NodeEvent struct {
+	Type EventType
+	Node *Node
+	Old  *Node
+}
+
+// PeerEvent represents an event concerning a peer in the cluster.
+type PeerEvent struct {
+	Type EventType
+	Peer *Peer
+	Old  *Peer
+}
+
+// Backend can create clients for all of the
+// primitive types that Kilo deals with, namely:
+// * nodes; and
+// * peers.
+type Backend interface {
+	Nodes() NodeBackend
+	Peers() PeerBackend
+}
+
+// NodeBackend can get nodes by name, init itself,
+// list the nodes that should be meshed,
+// set Kilo properties for a node,
+// clean up any changes applied to the backend,
+// and watch for changes to nodes.
+type NodeBackend interface {
+	CleanUp(string) error
+	Get(string) (*Node, error)
+	Init(<-chan struct{}) error
+	List() ([]*Node, error)
+	Set(string, *Node) error
+	Watch() <-chan *NodeEvent
+}
+
+// PeerBackend can get peers by name, init itself,
+// list the peers that should be in the mesh,
+// set fields for a peer,
+// clean up any changes applied to the backend,
+// and watch for changes to peers.
+type PeerBackend interface {
+	CleanUp(string) error
+	Get(string) (*Peer, error)
+	Init(<-chan struct{}) error
+	List() ([]*Peer, error)
+	Set(string, *Peer) error
+	Watch() <-chan *PeerEvent
+}

pkg/mesh/cni.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build linux
+
 package mesh
 
 import (

pkg/mesh/discoverips.go

@@ -0,0 +1,287 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package mesh
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"sort"
+
+	"github.com/vishvananda/netlink"
+)
+
+// getIP returns a private and public IP address for the local node.
+// It selects the private IP address in the following order:
+// - private IP to which hostname resolves
+// - private IP assigned to interface of default route
+// - private IP assigned to local interface
+// - nil if no private IP was found
+// It selects the public IP address in the following order:
+// - public IP to which hostname resolves
+// - public IP assigned to interface of default route
+// - public IP assigned to local interface
+// - private IP to which hostname resolves
+// - private IP assigned to interface of default route
+// - private IP assigned to local interface
+// - if no IP was found, return nil and an error.
+func getIP(hostname string, ignoreIfaces ...int) (*net.IPNet, *net.IPNet, error) {
+	ignore := make(map[string]struct{})
+	for i := range ignoreIfaces {
+		if ignoreIfaces[i] == 0 {
+			// Only ignore valid interfaces.
+			continue
+		}
+		iface, err := net.InterfaceByIndex(ignoreIfaces[i])
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to find interface %d: %v", ignoreIfaces[i], err)
+		}
+		ips, err := ipsForInterface(iface)
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, ip := range ips {
+			ignore[ip.String()] = struct{}{}
+			ignore[oneAddressCIDR(ip.IP).String()] = struct{}{}
+		}
+	}
+	var hostPriv, hostPub []*net.IPNet
+	{
+		// Check IPs to which hostname resolves first.
+		ips := ipsForHostname(hostname)
+		for _, ip := range ips {
+			ok, mask, err := assignedToInterface(ip)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to search locally assigned addresses: %v", err)
+			}
+			if !ok {
+				continue
+			}
+			ip.Mask = mask
+			if isPublic(ip.IP) {
+				hostPub = append(hostPub, ip)
+				continue
+			}
+			hostPriv = append(hostPriv, ip)
+		}
+		sortIPs(hostPriv)
+		sortIPs(hostPub)
+	}
+
+	var defaultPriv, defaultPub []*net.IPNet
+	{
+		// Check IPs on interface for default route next.
+		iface, err := defaultInterface()
+		if err != nil {
+			return nil, nil, err
+		}
+		ips, err := ipsForInterface(iface)
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, ip := range ips {
+			if isLocal(ip.IP) {
+				continue
+			}
+			if isPublic(ip.IP) {
+				defaultPub = append(defaultPub, ip)
+				continue
+			}
+			defaultPriv = append(defaultPriv, ip)
+		}
+		sortIPs(defaultPriv)
+		sortIPs(defaultPub)
+	}
+
+	var interfacePriv, interfacePub []*net.IPNet
+	{
+		// Finally look for IPs on all interfaces.
+		ips, err := ipsForAllInterfaces()
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, ip := range ips {
+			if isLocal(ip.IP) {
+				continue
+			}
+			if isPublic(ip.IP) {
+				interfacePub = append(interfacePub, ip)
+				continue
+			}
+			interfacePriv = append(interfacePriv, ip)
+		}
+		sortIPs(interfacePriv)
+		sortIPs(interfacePub)
+	}
+
+	var priv, pub, tmpPriv, tmpPub []*net.IPNet
+	tmpPriv = append(tmpPriv, hostPriv...)
+	tmpPriv = append(tmpPriv, defaultPriv...)
+	tmpPriv = append(tmpPriv, interfacePriv...)
+	tmpPub = append(tmpPub, hostPub...)
+	tmpPub = append(tmpPub, defaultPub...)
+	tmpPub = append(tmpPub, interfacePub...)
+	for i := range tmpPriv {
+		if _, ok := ignore[tmpPriv[i].String()]; ok {
+			continue
+		}
+		priv = append(priv, tmpPriv[i])
+	}
+	for i := range tmpPub {
+		if _, ok := ignore[tmpPub[i].String()]; ok {
+			continue
+		}
+		pub = append(pub, tmpPub[i])
+	}
+	if len(priv) == 0 && len(pub) == 0 {
+		return nil, nil, errors.New("no valid IP was found")
+	}
+	if len(priv) == 0 {
+		// If no private IPs were found, use nil.
+		priv = append(priv, nil)
+	}
+	if len(pub) == 0 {
+		pub = priv
+	}
+	return priv[0], pub[0], nil
+}
+
+func assignedToInterface(ip *net.IPNet) (bool, net.IPMask, error) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return false, nil, fmt.Errorf("failed to list interfaces: %v", err)
+	}
+	// Sort the links for stability.
+	sort.Slice(links, func(i, j int) bool {
+		return links[i].Attrs().Name < links[j].Attrs().Name
+	})
+	for _, link := range links {
+		addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+		if err != nil {
+			return false, nil, fmt.Errorf("failed to list addresses for %s: %v", link.Attrs().Name, err)
+		}
+		// Sort the IPs for stability.
+		sort.Slice(addrs, func(i, j int) bool {
+			return addrs[i].String() < addrs[j].String()
+		})
+		for i := range addrs {
+			if ip.IP.Equal(addrs[i].IP) {
+				return true, addrs[i].Mask, nil
+			}
+		}
+	}
+	return false, nil, nil
+}
+
+// ipsForHostname returns a slice of IPs to which the
+// given hostname resolves.
+func ipsForHostname(hostname string) []*net.IPNet {
+	if ip := net.ParseIP(hostname); ip != nil {
+		return []*net.IPNet{oneAddressCIDR(ip)}
+	}
+	ips, err := net.LookupIP(hostname)
+	if err != nil {
+		// Most likely the hostname is not resolvable.
+		return nil
+	}
+	nets := make([]*net.IPNet, len(ips))
+	for i := range ips {
+		nets[i] = oneAddressCIDR(ips[i])
+	}
+	return nets
+}
+
+// ipsForAllInterfaces returns a slice of IPs assigned to all the
+// interfaces on the host.
+func ipsForAllInterfaces() ([]*net.IPNet, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list interfaces: %v", err)
+	}
+	var nets []*net.IPNet
+	for _, iface := range ifaces {
+		ips, err := ipsForInterface(&iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
+		}
+		nets = append(nets, ips...)
+	}
+	return nets, nil
+}
+
+// ipsForInterface returns a slice of IPs assigned to the given interface.
+func ipsForInterface(iface *net.Interface) ([]*net.IPNet, error) {
+	link, err := netlink.LinkByIndex(iface.Index)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get link: %s", err)
+	}
+	addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
+	}
+	var ips []*net.IPNet
+	for _, a := range addrs {
+		if a.IPNet != nil {
+			ips = append(ips, a.IPNet)
+		}
+	}
+	return ips, nil
+}
+
+// interfacesForIP returns a slice of interfaces withthe given IP.
+func interfacesForIP(ip *net.IPNet) ([]net.Interface, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list interfaces: %v", err)
+	}
+	var interfaces []net.Interface
+	for _, iface := range ifaces {
+		ips, err := ipsForInterface(&iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
+		}
+		for i := range ips {
+			if ip.IP.Equal(ips[i].IP) {
+				interfaces = append(interfaces, iface)
+				break
+			}
+		}
+	}
+	if len(interfaces) == 0 {
+		return nil, fmt.Errorf("no interface has %s assigned", ip.String())
+	}
+	return interfaces, nil
+}
+
+// defaultInterface returns the interface for the default route of the host.
+func defaultInterface() (*net.Interface, error) {
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, route := range routes {
+		if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" || route.Dst.String() == "::/0" {
+			if route.LinkIndex <= 0 {
+				return nil, errors.New("failed to determine interface of route")
+			}
+			return net.InterfaceByIndex(route.LinkIndex)
+		}
+	}
+
+	return nil, errors.New("failed to find default route")
+}

pkg/mesh/graph.go

@@ -27,7 +27,7 @@ import (
 func (t *Topology) Dot() (string, error) {
 	g := gographviz.NewGraph()
 	g.Name = "kilo"
-	if err := g.AddAttr("kilo", string(gographviz.Label), graphEscape(t.subnet.String())); err != nil {
+	if err := g.AddAttr("kilo", string(gographviz.Label), graphEscape((&net.IPNet{IP: t.wireGuardCIDR.IP.Mask(t.wireGuardCIDR.Mask), Mask: t.wireGuardCIDR.Mask}).String())); err != nil {
 		return "", fmt.Errorf("failed to add label to graph")
 	}
 	if err := g.AddAttr("kilo", string(gographviz.LabelLOC), "t"); err != nil {
@@ -70,7 +70,11 @@ func (t *Topology) Dot() (string, error) {
 					return "", fmt.Errorf("failed to add rank to node")
 				}
 			}
-			if err := g.Nodes.Lookup[graphEscape(s.hostnames[j])].Attrs.Add(string(gographviz.Label), nodeLabel(s.location, s.hostnames[j], s.cidrs[j], s.privateIPs[j], wg, endpoint)); err != nil {
+			var priv net.IP
+			if s.privateIPs != nil {
+				priv = s.privateIPs[j]
+			}
+			if err := g.Nodes.Lookup[graphEscape(s.hostnames[j])].Attrs.Add(string(gographviz.Label), nodeLabel(s.location, s.hostnames[j], s.cidrs[j], priv, wg, endpoint)); err != nil {
 				return "", fmt.Errorf("failed to add label to node")
 			}
 		}
@@ -155,7 +159,9 @@ func nodeLabel(location, name string, cidr *net.IPNet, priv, wgIP net.IP, endpoi
 		location,
 		name,
 		cidr.String(),
-		priv.String(),
+	}
+	if priv != nil {
+		label = append(label, priv.String())
 	}
 	if wgIP != nil {
 		label = append(label, wgIP.String())
@@ -163,9 +169,9 @@ func nodeLabel(location, name string, cidr *net.IPNet, priv, wgIP net.IP, endpoi
 	if endpoint != nil {
 		label = append(label, endpoint.String())
 	}
-	return graphEscape(strings.Join(label, "\n"))
+	return graphEscape(strings.Join(label, "\\n"))
 }
 
 func peerLabel(peer *Peer) string {
-	return graphEscape(fmt.Sprintf("%s\n%s\n", peer.Name, peer.Endpoint.String()))
+	return graphEscape(fmt.Sprintf("%s\\n%s\n", peer.Name, peer.Endpoint.String()))
 }

pkg/mesh/ip.go

@@ -15,150 +15,10 @@
 package mesh
 
 import (
-	"errors"
-	"fmt"
 	"net"
 	"sort"
-
-	"github.com/vishvananda/netlink"
 )
 
-// getIP returns a private and public IP address for the local node.
-// It selects the private IP address in the following order:
-// - private IP to which hostname resolves
-// - private IP assigned to interface of default route
-// - private IP assigned to local interface
-// - public IP to which hostname resolves
-// - public IP assigned to interface of default route
-// - public IP assigned to local interface
-// It selects the public IP address in the following order:
-// - public IP to which hostname resolves
-// - public IP assigned to interface of default route
-// - public IP assigned to local interface
-// - private IP to which hostname resolves
-// - private IP assigned to interface of default route
-// - private IP assigned to local interface
-// - if no IP was found, return nil and an error.
-func getIP(hostname string, ignoreIfaces ...int) (*net.IPNet, *net.IPNet, error) {
-	ignore := make(map[string]struct{})
-	for i := range ignoreIfaces {
-		if ignoreIfaces[i] == 0 {
-			// Only ignore valid interfaces.
-			continue
-		}
-		iface, err := net.InterfaceByIndex(ignoreIfaces[i])
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to find interface %d: %v", ignoreIfaces[i], err)
-		}
-		ips, err := ipsForInterface(iface)
-		if err != nil {
-			return nil, nil, err
-		}
-		for _, ip := range ips {
-			ignore[ip.String()] = struct{}{}
-			ignore[oneAddressCIDR(ip.IP).String()] = struct{}{}
-		}
-	}
-	var hostPriv, hostPub []*net.IPNet
-	{
-		// Check IPs to which hostname resolves first.
-		ips := ipsForHostname(hostname)
-		for _, ip := range ips {
-			ok, mask, err := assignedToInterface(ip)
-			if err != nil {
-				return nil, nil, fmt.Errorf("failed to search locally assigned addresses: %v", err)
-			}
-			if !ok {
-				continue
-			}
-			ip.Mask = mask
-			if isPublic(ip.IP) {
-				hostPub = append(hostPub, ip)
-				continue
-			}
-			hostPriv = append(hostPriv, ip)
-		}
-		sortIPs(hostPriv)
-		sortIPs(hostPub)
-	}
-
-	var defaultPriv, defaultPub []*net.IPNet
-	{
-		// Check IPs on interface for default route next.
-		iface, err := defaultInterface()
-		if err != nil {
-			return nil, nil, err
-		}
-		ips, err := ipsForInterface(iface)
-		if err != nil {
-			return nil, nil, err
-		}
-		for _, ip := range ips {
-			if isLocal(ip.IP) {
-				continue
-			}
-			if isPublic(ip.IP) {
-				defaultPub = append(defaultPub, ip)
-				continue
-			}
-			defaultPriv = append(defaultPriv, ip)
-		}
-		sortIPs(defaultPriv)
-		sortIPs(defaultPub)
-	}
-
-	var interfacePriv, interfacePub []*net.IPNet
-	{
-		// Finally look for IPs on all interfaces.
-		ips, err := ipsForAllInterfaces()
-		if err != nil {
-			return nil, nil, err
-		}
-		for _, ip := range ips {
-			if isLocal(ip.IP) {
-				continue
-			}
-			if isPublic(ip.IP) {
-				interfacePub = append(interfacePub, ip)
-				continue
-			}
-			interfacePriv = append(interfacePriv, ip)
-		}
-		sortIPs(interfacePriv)
-		sortIPs(interfacePub)
-	}
-
-	var priv, pub, tmpPriv, tmpPub []*net.IPNet
-	tmpPriv = append(tmpPriv, hostPriv...)
-	tmpPriv = append(tmpPriv, defaultPriv...)
-	tmpPriv = append(tmpPriv, interfacePriv...)
-	tmpPub = append(tmpPub, hostPub...)
-	tmpPub = append(tmpPub, defaultPub...)
-	tmpPub = append(tmpPub, interfacePub...)
-	for i := range tmpPriv {
-		if _, ok := ignore[tmpPriv[i].String()]; ok {
-			continue
-		}
-		priv = append(priv, tmpPriv[i])
-	}
-	for i := range tmpPub {
-		if _, ok := ignore[tmpPub[i].String()]; ok {
-			continue
-		}
-		pub = append(pub, tmpPub[i])
-	}
-	if len(priv) == 0 && len(pub) == 0 {
-		return nil, nil, errors.New("no valid IP was found")
-	}
-	if len(priv) == 0 {
-		priv = pub
-	}
-	if len(pub) == 0 {
-		pub = priv
-	}
-	return priv[0], pub[0], nil
-}
-
 // sortIPs sorts IPs so the result is stable.
 // It will first sort IPs by type, to prefer selecting
 // IPs of the same type, and then by value.
@@ -175,33 +35,6 @@ func sortIPs(ips []*net.IPNet) {
 	})
 }
 
-func assignedToInterface(ip *net.IPNet) (bool, net.IPMask, error) {
-	links, err := netlink.LinkList()
-	if err != nil {
-		return false, nil, fmt.Errorf("failed to list interfaces: %v", err)
-	}
-	// Sort the links for stability.
-	sort.Slice(links, func(i, j int) bool {
-		return links[i].Attrs().Name < links[j].Attrs().Name
-	})
-	for _, link := range links {
-		addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
-		if err != nil {
-			return false, nil, fmt.Errorf("failed to list addresses for %s: %v", link.Attrs().Name, err)
-		}
-		// Sort the IPs for stability.
-		sort.Slice(addrs, func(i, j int) bool {
-			return addrs[i].String() < addrs[j].String()
-		})
-		for i := range addrs {
-			if ip.IP.Equal(addrs[i].IP) {
-				return true, addrs[i].Mask, nil
-			}
-		}
-	}
-	return false, nil, nil
-}
-
 func isLocal(ip net.IP) bool {
 	return ip.IsLoopback() || ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()
 }
@@ -209,12 +42,12 @@ func isLocal(ip net.IP) bool {
 func isPublic(ip net.IP) bool {
 	// Check RFC 1918 addresses.
 	if ip4 := ip.To4(); ip4 != nil {
-		switch true {
+		switch {
 		// Check for 10.0.0.0/8.
 		case ip4[0] == 10:
 			return false
 		// Check for 172.16.0.0/12.
-		case ip4[0] == 172 && ip4[1]&0xf0 == 0x01:
+		case ip4[0] == 172 && ip4[1]&0xf0 == 0x10:
 			return false
 		// Check for 192.168.0.0/16.
 		case ip4[0] == 192 && ip4[1] == 168:
@@ -225,7 +58,7 @@ func isPublic(ip net.IP) bool {
 	}
 	// Check RFC 4193 addresses.
 	if len(ip) == net.IPv6len {
-		switch true {
+		switch {
 		// Check for fd00::/8.
 		case ip[0] == 0xfd && ip[1] == 0x00:
 			return false
@@ -236,105 +69,6 @@ func isPublic(ip net.IP) bool {
 	return false
 }
 
-// ipsForHostname returns a slice of IPs to which the
-// given hostname resolves.
-func ipsForHostname(hostname string) []*net.IPNet {
-	if ip := net.ParseIP(hostname); ip != nil {
-		return []*net.IPNet{oneAddressCIDR(ip)}
-	}
-	ips, err := net.LookupIP(hostname)
-	if err != nil {
-		// Most likely the hostname is not resolvable.
-		return nil
-	}
-	nets := make([]*net.IPNet, len(ips))
-	for i := range ips {
-		nets[i] = oneAddressCIDR(ips[i])
-	}
-	return nets
-}
-
-// ipsForAllInterfaces returns a slice of IPs assigned to all the
-// interfaces on the host.
-func ipsForAllInterfaces() ([]*net.IPNet, error) {
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list interfaces: %v", err)
-	}
-	var nets []*net.IPNet
-	for _, iface := range ifaces {
-		ips, err := ipsForInterface(&iface)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
-		}
-		nets = append(nets, ips...)
-	}
-	return nets, nil
-}
-
-// ipsForInterface returns a slice of IPs assigned to the given interface.
-func ipsForInterface(iface *net.Interface) ([]*net.IPNet, error) {
-	link, err := netlink.LinkByIndex(iface.Index)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get link: %s", err)
-	}
-	addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
-	if err != nil {
-		return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
-	}
-	var ips []*net.IPNet
-	for _, a := range addrs {
-		if a.IPNet != nil {
-			ips = append(ips, a.IPNet)
-		}
-	}
-	return ips, nil
-}
-
-// interfacesForIP returns a slice of interfaces withthe given IP.
-func interfacesForIP(ip *net.IPNet) ([]net.Interface, error) {
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list interfaces: %v", err)
-	}
-	var interfaces []net.Interface
-	for _, iface := range ifaces {
-		ips, err := ipsForInterface(&iface)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list addresses for %s: %v", iface.Name, err)
-		}
-		for i := range ips {
-			if ip.IP.Equal(ips[i].IP) {
-				interfaces = append(interfaces, iface)
-				break
-			}
-		}
-	}
-	if len(interfaces) == 0 {
-		return nil, fmt.Errorf("no interface has %s assigned", ip.String())
-	}
-	return interfaces, nil
-}
-
-// defaultInterface returns the interface for the default route of the host.
-func defaultInterface() (*net.Interface, error) {
-	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, route := range routes {
-		if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" || route.Dst.String() == "::/0" {
-			if route.LinkIndex <= 0 {
-				return nil, errors.New("failed to determine interface of route")
-			}
-			return net.InterfaceByIndex(route.LinkIndex)
-		}
-	}
-
-	return nil, errors.New("failed to find default route")
-}
-
 type allocator struct {
 	bits    int
 	ones    int

pkg/mesh/ip_test.go

@@ -127,3 +127,46 @@ func TestSortIPs(t *testing.T) {
 		}
 	}
 }
+
+func TestIsPublic(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		ip   net.IP
+		out  bool
+	}{
+		{
+			name: "10/8",
+			ip:   net.ParseIP("10.0.0.1"),
+			out:  false,
+		},
+		{
+			name: "172.16/12",
+			ip:   net.ParseIP("172.16.0.0"),
+			out:  false,
+		},
+		{
+			name: "172.16/12 random",
+			ip:   net.ParseIP("172.24.135.46"),
+			out:  false,
+		},
+		{
+			name: "below 172.16/12",
+			ip:   net.ParseIP("172.15.255.255"),
+			out:  true,
+		},
+		{
+			name: "above 172.16/12",
+			ip:   net.ParseIP("172.160.255.255"),
+			out:  true,
+		},
+		{
+			name: "192.168/16",
+			ip:   net.ParseIP("192.168.0.0"),
+			out:  false,
+		},
+	} {
+		if isPublic(tc.ip) != tc.out {
+			t.Errorf("test case %q: expected %t, got %t", tc.name, tc.out, !tc.out)
+		}
+	}
+}

pkg/mesh/mesh.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build linux
+
 package mesh
 
 import (
@@ -35,142 +37,15 @@ import (
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
-const resyncPeriod = 30 * time.Second
-
 const (
-	// KiloPath is the directory where Kilo stores its configuration.
+	// kiloPath is the directory where Kilo stores its configuration.
-	KiloPath = "/var/lib/kilo"
+	kiloPath = "/var/lib/kilo"
-	// PrivateKeyPath is the filepath where the WireGuard private key is stored.
+	// privateKeyPath is the filepath where the WireGuard private key is stored.
-	PrivateKeyPath = KiloPath + "/key"
+	privateKeyPath = kiloPath + "/key"
-	// ConfPath is the filepath where the WireGuard configuration is stored.
+	// confPath is the filepath where the WireGuard configuration is stored.
-	ConfPath = KiloPath + "/conf"
+	confPath = kiloPath + "/conf"
-	// DefaultKiloInterface is the default iterface created and used by Kilo.
-	DefaultKiloInterface = "kilo0"
-	// DefaultKiloPort is the default UDP port Kilo uses.
-	DefaultKiloPort = 51820
-	// DefaultCNIPath is the default path to the CNI config file.
-	DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist"
 )
 
-// DefaultKiloSubnet is the default CIDR for Kilo.
-var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}}
-
-// Granularity represents the abstraction level at which the network
-// should be meshed.
-type Granularity string
-
-const (
-	// LogicalGranularity indicates that the network should create
-	// a mesh between logical locations, e.g. data-centers, but not between
-	// all nodes within a single location.
-	LogicalGranularity Granularity = "location"
-	// FullGranularity indicates that the network should create
-	// a mesh between every node.
-	FullGranularity Granularity = "full"
-)
-
-// Node represents a node in the network.
-type Node struct {
-	Endpoint   *wireguard.Endpoint
-	Key        []byte
-	InternalIP *net.IPNet
-	// LastSeen is a Unix time for the last time
-	// the node confirmed it was live.
-	LastSeen int64
-	// Leader is a suggestion to Kilo that
-	// the node wants to lead its segment.
-	Leader              bool
-	Location            string
-	Name                string
-	PersistentKeepalive int
-	Subnet              *net.IPNet
-	WireGuardIP         *net.IPNet
-}
-
-// Ready indicates whether or not the node is ready.
-func (n *Node) Ready() bool {
-	// Nodes that are not leaders will not have WireGuardIPs, so it is not required.
-	return n != nil && n.Endpoint != nil && !(n.Endpoint.IP == nil && n.Endpoint.DNS == "") && n.Endpoint.Port != 0 && n.Key != nil && n.InternalIP != nil && n.Subnet != nil && time.Now().Unix()-n.LastSeen < int64(resyncPeriod)*2/int64(time.Second)
-}
-
-// Peer represents a peer in the network.
-type Peer struct {
-	wireguard.Peer
-	Name string
-}
-
-// Ready indicates whether or not the peer is ready.
-// Peers can have empty endpoints because they may not have an
-// IP, for example if they are behind a NAT, and thus
-// will not declare their endpoint and instead allow it to be
-// discovered.
-func (p *Peer) Ready() bool {
-	return p != nil && p.AllowedIPs != nil && len(p.AllowedIPs) != 0 && p.PublicKey != nil
-}
-
-// EventType describes what kind of an action an event represents.
-type EventType string
-
-const (
-	// AddEvent represents an action where an item was added.
-	AddEvent EventType = "add"
-	// DeleteEvent represents an action where an item was removed.
-	DeleteEvent EventType = "delete"
-	// UpdateEvent represents an action where an item was updated.
-	UpdateEvent EventType = "update"
-)
-
-// NodeEvent represents an event concerning a node in the cluster.
-type NodeEvent struct {
-	Type EventType
-	Node *Node
-	Old  *Node
-}
-
-// PeerEvent represents an event concerning a peer in the cluster.
-type PeerEvent struct {
-	Type EventType
-	Peer *Peer
-	Old  *Peer
-}
-
-// Backend can create clients for all of the
-// primitive types that Kilo deals with, namely:
-// * nodes; and
-// * peers.
-type Backend interface {
-	Nodes() NodeBackend
-	Peers() PeerBackend
-}
-
-// NodeBackend can get nodes by name, init itself,
-// list the nodes that should be meshed,
-// set Kilo properties for a node,
-// clean up any changes applied to the backend,
-// and watch for changes to nodes.
-type NodeBackend interface {
-	CleanUp(string) error
-	Get(string) (*Node, error)
-	Init(<-chan struct{}) error
-	List() ([]*Node, error)
-	Set(string, *Node) error
-	Watch() <-chan *NodeEvent
-}
-
-// PeerBackend can get peers by name, init itself,
-// list the peers that should be in the mesh,
-// set fields for a peer,
-// clean up any changes applied to the backend,
-// and watch for changes to peers.
-type PeerBackend interface {
-	CleanUp(string) error
-	Get(string) (*Peer, error)
-	Init(<-chan struct{}) error
-	List() ([]*Peer, error)
-	Set(string, *Peer) error
-	Watch() <-chan *PeerEvent
-}
-
 // Mesh is able to create Kilo network meshes.
 type Mesh struct {
 	Backend
@@ -190,13 +65,14 @@ type Mesh struct {
 	priv         []byte
 	privIface    int
 	pub          []byte
+	resyncPeriod time.Duration
 	stop         chan struct{}
 	subnet       *net.IPNet
 	table        *route.Table
 	wireGuardIP  *net.IPNet
 
 	// nodes and peers are mutable fields in the struct
-	// and needs to be guarded.
+	// and need to be guarded.
 	nodes map[string]*Node
 	peers map[string]*Peer
 	mu    sync.Mutex
@@ -210,11 +86,11 @@ type Mesh struct {
 }
 
 // New returns a new Mesh instance.
-func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, logger log.Logger) (*Mesh, error) {
+func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port uint32, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, createIface bool, resyncPeriod time.Duration, logger log.Logger) (*Mesh, error) {
-	if err := os.MkdirAll(KiloPath, 0700); err != nil {
+	if err := os.MkdirAll(kiloPath, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create directory to store configuration: %v", err)
 	}
-	private, err := ioutil.ReadFile(PrivateKeyPath)
+	private, err := ioutil.ReadFile(privateKeyPath)
 	private = bytes.Trim(private, "\n")
 	if err != nil {
 		level.Warn(logger).Log("msg", "no private key found on disk; generating one now")
@@ -226,34 +102,49 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(PrivateKeyPath, private, 0600); err != nil {
+	if err := ioutil.WriteFile(privateKeyPath, private, 0600); err != nil {
 		return nil, fmt.Errorf("failed to write private key to disk: %v", err)
 	}
 	cniIndex, err := cniDeviceIndex()
 	if err != nil {
 		return nil, fmt.Errorf("failed to query netlink for CNI device: %v", err)
 	}
-	privateIP, publicIP, err := getIP(hostname, enc.Index(), cniIndex)
+	var kiloIface int
+	if createIface {
+		kiloIface, _, err = wireguard.New(iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create WireGuard interface: %v", err)
+		}
+	} else {
+		link, err := netlink.LinkByName(iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get interface index: %v", err)
+		}
+		kiloIface = link.Attrs().Index
+	}
+	privateIP, publicIP, err := getIP(hostname, kiloIface, enc.Index(), cniIndex)
 	if err != nil {
 		return nil, fmt.Errorf("failed to find public IP: %v", err)
 	}
+	var privIface int
+	if privateIP != nil {
 		ifaces, err := interfacesForIP(privateIP)
 		if err != nil {
 			return nil, fmt.Errorf("failed to find interface for private IP: %v", err)
 		}
-	privIface := ifaces[0].Index
+		privIface = ifaces[0].Index
-	kiloIface, _, err := wireguard.New(iface)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create WireGuard interface: %v", err)
-	}
 		if enc.Strategy() != encapsulation.Never {
 			if err := enc.Init(privIface); err != nil {
 				return nil, fmt.Errorf("failed to initialize encapsulator: %v", err)
 			}
 		}
 		level.Debug(logger).Log("msg", fmt.Sprintf("using %s as the private IP address", privateIP.String()))
+	} else {
+		enc = encapsulation.Noop(enc.Strategy())
+		level.Debug(logger).Log("msg", "running without a private IP address")
+	}
 	level.Debug(logger).Log("msg", fmt.Sprintf("using %s as the public IP address", publicIP.String()))
-	ipTables, err := iptables.New()
+	ipTables, err := iptables.New(iptables.WithLogger(log.With(logger, "component", "iptables")), iptables.WithResyncPeriod(resyncPeriod))
 	if err != nil {
 		return nil, fmt.Errorf("failed to IP tables controller: %v", err)
 	}
@@ -275,6 +166,7 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 		priv:         private,
 		privIface:    privIface,
 		pub:          public,
+		resyncPeriod: resyncPeriod,
 		local:        local,
 		stop:         make(chan struct{}),
 		subnet:       subnet,
@@ -344,7 +236,8 @@ func (m *Mesh) Run() error {
 		}
 	}()
 	defer m.cleanUp()
-	t := time.NewTimer(resyncPeriod)
+	resync := time.NewTimer(m.resyncPeriod)
+	checkIn := time.NewTimer(checkInPeriod)
 	nw := m.Nodes().Watch()
 	pw := m.Peers().Watch()
 	var ne *NodeEvent
@@ -355,13 +248,15 @@ func (m *Mesh) Run() error {
 			m.syncNodes(ne)
 		case pe = <-pw:
 			m.syncPeers(pe)
-		case <-t.C:
+		case <-checkIn.C:
 			m.checkIn()
+			checkIn.Reset(checkInPeriod)
+		case <-resync.C:
 			if m.cni {
 				m.updateCNIConfig()
 			}
 			m.applyTopology()
-			t.Reset(resyncPeriod)
+			resync.Reset(m.resyncPeriod)
 		case <-m.stop:
 			return nil
 		}
@@ -379,14 +274,10 @@ func (m *Mesh) syncNodes(e *NodeEvent) {
 	var diff bool
 	m.mu.Lock()
 	if !e.Node.Ready() {
-		level.Debug(logger).Log("msg", "received incomplete node", "node", e.Node)
+		// Trace non ready nodes with their presence in the mesh.
-		// An existing node is no longer valid
+		_, ok := m.nodes[e.Node.Name]
-		// so remove it from the mesh.
+		level.Debug(logger).Log("msg", "received non ready node", "node", e.Node, "in-mesh", ok)
-		if _, ok := m.nodes[e.Node.Name]; ok {
-			level.Info(logger).Log("msg", "node is no longer ready", "node", e.Node)
-			diff = true
 	}
-	} else {
 	switch e.Type {
 	case AddEvent:
 		fallthrough
@@ -401,7 +292,6 @@ func (m *Mesh) syncNodes(e *NodeEvent) {
 		delete(m.nodes, e.Node.Name)
 		diff = true
 	}
-	}
 	m.mu.Unlock()
 	if diff {
 		level.Info(logger).Log("node", e.Node)
@@ -417,14 +307,10 @@ func (m *Mesh) syncPeers(e *PeerEvent) {
 	// Peers are indexed by public key.
 	key := string(e.Peer.PublicKey)
 	if !e.Peer.Ready() {
-		level.Debug(logger).Log("msg", "received incomplete peer", "peer", e.Peer)
+		// Trace non ready peer with their presence in the mesh.
-		// An existing peer is no longer valid
+		_, ok := m.peers[key]
-		// so remove it from the mesh.
+		level.Debug(logger).Log("msg", "received non ready peer", "peer", e.Peer, "in-mesh", ok)
-		if _, ok := m.peers[key]; ok {
-			level.Info(logger).Log("msg", "peer is no longer ready", "peer", e.Peer)
-			diff = true
 	}
-	} else {
 	switch e.Type {
 	case AddEvent:
 		fallthrough
@@ -441,7 +327,6 @@ func (m *Mesh) syncPeers(e *PeerEvent) {
 		delete(m.peers, key)
 		diff = true
 	}
-	}
 	m.mu.Unlock()
 	if diff {
 		level.Info(logger).Log("peer", e.Peer)
@@ -476,7 +361,7 @@ func (m *Mesh) handleLocal(n *Node) {
 	if n.Endpoint == nil || (n.Endpoint.DNS == "" && n.Endpoint.IP == nil) {
 		n.Endpoint = &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: m.externalIP.IP}, Port: m.port}
 	}
-	if n.InternalIP == nil {
+	if n.InternalIP == nil && !n.NoInternalIP {
 		n.InternalIP = m.internalIP
 	}
 	// Compare the given node to the calculated local node.
@@ -485,6 +370,7 @@ func (m *Mesh) handleLocal(n *Node) {
 	local := &Node{
 		Endpoint:            n.Endpoint,
 		Key:                 m.pub,
+		NoInternalIP:        n.NoInternalIP,
 		InternalIP:          n.InternalIP,
 		LastSeen:            time.Now().Unix(),
 		Leader:              n.Leader,
@@ -493,6 +379,9 @@ func (m *Mesh) handleLocal(n *Node) {
 		PersistentKeepalive: n.PersistentKeepalive,
 		Subnet:              n.Subnet,
 		WireGuardIP:         m.wireGuardIP,
+		DiscoveredEndpoints: n.DiscoveredEndpoints,
+		AllowedLocationIPs:  n.AllowedLocationIPs,
+		Granularity:         m.granularity,
 	}
 	if !nodesAreEqual(n, local) {
 		level.Debug(m.logger).Log("msg", "local node differs from backend")
@@ -532,12 +421,12 @@ func (m *Mesh) applyTopology() {
 	nodes := make(map[string]*Node)
 	var readyNodes float64
 	for k := range m.nodes {
+		m.nodes[k].Granularity = m.granularity
 		if !m.nodes[k].Ready() {
 			continue
 		}
-		// Make a shallow copy of the node.
+		// Make it point to the node without copy.
-		node := *m.nodes[k]
+		nodes[k] = m.nodes[k]
-		nodes[k] = &node
 		readyNodes++
 	}
 	// Ensure only ready nodes are considered.
@@ -547,9 +436,8 @@ func (m *Mesh) applyTopology() {
 		if !m.peers[k].Ready() {
 			continue
 		}
-		// Make a shallow copy of the peer.
+		// Make it point the peer without copy.
-		peer := *m.peers[k]
+		peers[k] = m.peers[k]
-		peers[k] = &peer
 		readyPeers++
 	}
 	m.nodesGuage.Set(readyNodes)
@@ -566,22 +454,32 @@ func (m *Mesh) applyTopology() {
 		return
 	}
 	// Find the old configuration.
-	oldConfRaw, err := wireguard.ShowConf(link.Attrs().Name)
+	oldConfDump, err := wireguard.ShowDump(link.Attrs().Name)
 	if err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	oldConf := wireguard.Parse(oldConfRaw)
+	oldConf, err := wireguard.ParseDump(oldConfDump)
-	updateNATEndpoints(nodes, peers, oldConf)
+	if err != nil {
-	t, err := NewTopology(nodes, peers, m.granularity, m.hostname, nodes[m.hostname].Endpoint.Port, m.priv, m.subnet, nodes[m.hostname].PersistentKeepalive)
+		level.Error(m.logger).Log("error", err)
+		m.errorCounter.WithLabelValues("apply").Inc()
+		return
+	}
+	natEndpoints := discoverNATEndpoints(nodes, peers, oldConf, m.logger)
+	nodes[m.hostname].DiscoveredEndpoints = natEndpoints
+	t, err := NewTopology(nodes, peers, m.granularity, m.hostname, nodes[m.hostname].Endpoint.Port, m.priv, m.subnet, nodes[m.hostname].PersistentKeepalive, m.logger)
 	if err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
 	// Update the node's WireGuard IP.
+	if t.leader {
 		m.wireGuardIP = t.wireGuardCIDR
+	} else {
+		m.wireGuardIP = nil
+	}
 	conf := t.Conf()
 	buf, err := conf.Bytes()
 	if err != nil {
@@ -589,21 +487,21 @@ func (m *Mesh) applyTopology() {
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	if err := ioutil.WriteFile(ConfPath, buf, 0600); err != nil {
+	if err := ioutil.WriteFile(confPath, buf, 0600); err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
-	var ipRules []iptables.Rule
+	ipRules := t.Rules(m.cni)
-	if m.cni {
-		ipRules = append(ipRules, t.Rules(m.cni)...)
-	}
 	// If we are handling local routes, ensure the local
 	// tunnel has an IP address and IPIP traffic is allowed.
 	if m.enc.Strategy() != encapsulation.Never && m.local {
 		var cidrs []*net.IPNet
 		for _, s := range t.segments {
-			if s.location == nodes[m.hostname].Location {
+			// If the location prefix is not logicalLocation, but nodeLocation,
+			// we don't need to set any extra rules for encapsulation anyways
+			// because traffic will go over WireGuard.
+			if s.location == logicalLocationPrefix+nodes[m.hostname].Location {
 				for i := range s.privateIPs {
 					cidrs = append(cidrs, oneAddressCIDR(s.privateIPs[i]))
 				}
@@ -636,7 +534,7 @@ func (m *Mesh) applyTopology() {
 		equal := conf.Equal(oldConf)
 		if !equal {
 			level.Info(m.logger).Log("msg", "WireGuard configurations are different")
-			if err := wireguard.SetConf(link.Attrs().Name, ConfPath); err != nil {
+			if err := wireguard.SetConf(link.Attrs().Name, confPath); err != nil {
 				level.Error(m.logger).Log("error", err)
 				m.errorCounter.WithLabelValues("apply").Inc()
 				return
@@ -691,7 +589,7 @@ func (m *Mesh) cleanUp() {
 		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up routes: %v", err))
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}
-	if err := os.Remove(ConfPath); err != nil {
+	if err := os.Remove(confPath); err != nil {
 		level.Error(m.logger).Log("error", fmt.Sprintf("failed to delete configuration file: %v", err))
 		m.errorCounter.WithLabelValues("cleanUp").Inc()
 	}
@@ -770,32 +668,21 @@ func isSelf(hostname string, node *Node) bool {
 }
 
 func nodesAreEqual(a, b *Node) bool {
-	if !(a != nil) == (b != nil) {
+	if (a != nil) != (b != nil) {
 		return false
 	}
 	if a == b {
 		return true
 	}
-	if !(a.Endpoint != nil) == (b.Endpoint != nil) {
-		return false
-	}
-	if a.Endpoint != nil {
-		if a.Endpoint.Port != b.Endpoint.Port {
-			return false
-		}
 	// Check the DNS name first since this package
 	// is doing the DNS resolution.
-		if a.Endpoint.DNS != b.Endpoint.DNS {
+	if !a.Endpoint.Equal(b.Endpoint, true) {
 		return false
 	}
-		if a.Endpoint.DNS == "" && !a.Endpoint.IP.Equal(b.Endpoint.IP) {
-			return false
-		}
-	}
 	// Ignore LastSeen when comparing equality we want to check if the nodes are
 	// equivalent. However, we do want to check if LastSeen has transitioned
 	// between valid and invalid.
-	return string(a.Key) == string(b.Key) && ipNetsEqual(a.WireGuardIP, b.WireGuardIP) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready() && a.PersistentKeepalive == b.PersistentKeepalive
+	return string(a.Key) == string(b.Key) && ipNetsEqual(a.WireGuardIP, b.WireGuardIP) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready() && a.PersistentKeepalive == b.PersistentKeepalive && discoveredEndpointsAreEqual(a.DiscoveredEndpoints, b.DiscoveredEndpoints) && ipNetSlicesEqual(a.AllowedLocationIPs, b.AllowedLocationIPs) && a.Granularity == b.Granularity
 }
 
 func peersAreEqual(a, b *Peer) bool {
@@ -805,22 +692,11 @@ func peersAreEqual(a, b *Peer) bool {
 	if a == b {
 		return true
 	}
-	if !(a.Endpoint != nil) == (b.Endpoint != nil) {
-		return false
-	}
-	if a.Endpoint != nil {
-		if a.Endpoint.Port != b.Endpoint.Port {
-			return false
-		}
 	// Check the DNS name first since this package
 	// is doing the DNS resolution.
-		if a.Endpoint.DNS != b.Endpoint.DNS {
+	if !a.Endpoint.Equal(b.Endpoint, true) {
 		return false
 	}
-		if a.Endpoint.DNS == "" && !a.Endpoint.IP.Equal(b.Endpoint.IP) {
-			return false
-		}
-	}
 	if len(a.AllowedIPs) != len(b.AllowedIPs) {
 		return false
 	}
@@ -845,6 +721,18 @@ func ipNetsEqual(a, b *net.IPNet) bool {
 	return a.IP.Equal(b.IP)
 }
 
+func ipNetSlicesEqual(a, b []*net.IPNet) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if !ipNetsEqual(a[i], b[i]) {
+			return false
+		}
+	}
+	return true
+}
+
 func subnetsEqual(a, b *net.IPNet) bool {
 	if a == nil && b == nil {
 		return true
@@ -864,6 +752,24 @@ func subnetsEqual(a, b *net.IPNet) bool {
 	return true
 }
 
+func discoveredEndpointsAreEqual(a, b map[string]*wireguard.Endpoint) bool {
+	if a == nil && b == nil {
+		return true
+	}
+	if (a != nil) != (b != nil) {
+		return false
+	}
+	if len(a) != len(b) {
+		return false
+	}
+	for k := range a {
+		if !a[k].Equal(b[k], false) {
+			return false
+		}
+	}
+	return true
+}
+
 func linkByIndex(index int) (netlink.Link, error) {
 	link, err := netlink.LinkByIndex(index)
 	if err != nil {
@@ -872,21 +778,28 @@ func linkByIndex(index int) (netlink.Link, error) {
 	return link, nil
 }
 
-// updateNATEndpoints ensures that nodes and peers behind NAT update
+// discoverNATEndpoints uses the node's WireGuard configuration to returns a list of the most recently discovered endpoints for all nodes and peers behind NAT so that they can roam.
-// their endpoints from the WireGuard configuration so they can roam.
+func discoverNATEndpoints(nodes map[string]*Node, peers map[string]*Peer, conf *wireguard.Conf, logger log.Logger) map[string]*wireguard.Endpoint {
-func updateNATEndpoints(nodes map[string]*Node, peers map[string]*Peer, conf *wireguard.Conf) {
+	natEndpoints := make(map[string]*wireguard.Endpoint)
 	keys := make(map[string]*wireguard.Peer)
 	for i := range conf.Peers {
 		keys[string(conf.Peers[i].PublicKey)] = conf.Peers[i]
 	}
 	for _, n := range nodes {
 		if peer, ok := keys[string(n.Key)]; ok && n.PersistentKeepalive > 0 {
-			n.Endpoint = peer.Endpoint
+			level.Debug(logger).Log("msg", "WireGuard Update NAT Endpoint", "node", n.Name, "endpoint", peer.Endpoint, "former-endpoint", n.Endpoint, "same", n.Endpoint.Equal(peer.Endpoint, false), "latest-handshake", peer.LatestHandshake)
+			if (peer.LatestHandshake != time.Time{}) {
+				natEndpoints[string(n.Key)] = peer.Endpoint
+			}
 		}
 	}
 	for _, p := range peers {
 		if peer, ok := keys[string(p.PublicKey)]; ok && p.PersistentKeepalive > 0 {
-			p.Endpoint = peer.Endpoint
+			if (peer.LatestHandshake != time.Time{}) {
+				natEndpoints[string(p.PublicKey)] = peer.Endpoint
 			}
 		}
+	}
+	level.Debug(logger).Log("msg", "Discovered WireGuard NAT Endpoints", "DiscoveredEndpoints", natEndpoints)
+	return natEndpoints
 }

pkg/mesh/routes.go

@@ -0,0 +1,288 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package mesh
+
+import (
+	"net"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/squat/kilo/pkg/encapsulation"
+	"github.com/squat/kilo/pkg/iptables"
+)
+
+const kiloTableIndex = 1107
+
+// Routes generates a slice of routes for a given Topology.
+func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface int, local bool, enc encapsulation.Encapsulator) ([]*netlink.Route, []*netlink.Rule) {
+	var routes []*netlink.Route
+	var rules []*netlink.Rule
+	if !t.leader {
+		// Find the GW for this segment.
+		// This will be the an IP of the leader.
+		// In an IPIP encapsulated mesh it is the leader's private IP.
+		var gw net.IP
+		for _, segment := range t.segments {
+			if segment.location == t.location {
+				gw = enc.Gw(segment.endpoint.IP, segment.privateIPs[segment.leader], segment.cidrs[segment.leader])
+				break
+			}
+		}
+		for _, segment := range t.segments {
+			// First, add a route to the WireGuard IP of the segment.
+			routes = append(routes, encapsulateRoute(&netlink.Route{
+				Dst:       oneAddressCIDR(segment.wireGuardIP),
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        gw,
+				LinkIndex: privIface,
+				Protocol:  unix.RTPROT_STATIC,
+			}, enc.Strategy(), t.privateIP, tunlIface))
+			// Add routes for the current segment if local is true.
+			if segment.location == t.location {
+				if local {
+					for i := range segment.cidrs {
+						// Don't add routes for the local node.
+						if segment.privateIPs[i].Equal(t.privateIP.IP) {
+							continue
+						}
+						routes = append(routes, encapsulateRoute(&netlink.Route{
+							Dst:       segment.cidrs[i],
+							Flags:     int(netlink.FLAG_ONLINK),
+							Gw:        segment.privateIPs[i],
+							LinkIndex: privIface,
+							Protocol:  unix.RTPROT_STATIC,
+						}, enc.Strategy(), t.privateIP, tunlIface))
+						// Encapsulate packets from the host's Pod subnet headed
+						// to private IPs.
+						if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
+							routes = append(routes, &netlink.Route{
+								Dst:       oneAddressCIDR(segment.privateIPs[i]),
+								Flags:     int(netlink.FLAG_ONLINK),
+								Gw:        segment.privateIPs[i],
+								LinkIndex: tunlIface,
+								Protocol:  unix.RTPROT_STATIC,
+								Table:     kiloTableIndex,
+							})
+							rules = append(rules, defaultRule(&netlink.Rule{
+								Src:   t.subnet,
+								Dst:   oneAddressCIDR(segment.privateIPs[i]),
+								Table: kiloTableIndex,
+							}))
+						}
+					}
+				}
+				continue
+			}
+			for i := range segment.cidrs {
+				// Add routes to the Pod CIDRs of nodes in other segments.
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       segment.cidrs[i],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
+			for i := range segment.privateIPs {
+				// Add routes to the private IPs of nodes in other segments.
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       oneAddressCIDR(segment.privateIPs[i]),
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
+			// For segments / locations other than the location of this instance of kg,
+			// we need to set routes for allowed location IPs over the leader in the current location.
+			for i := range segment.allowedLocationIPs {
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       segment.allowedLocationIPs[i],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
+		}
+		// Add routes for the allowed IPs of peers.
+		for _, peer := range t.peers {
+			for i := range peer.AllowedIPs {
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       peer.AllowedIPs[i],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
+		}
+		return routes, rules
+	}
+	for _, segment := range t.segments {
+		// Add routes for the current segment if local is true.
+		if segment.location == t.location {
+			// If the local node does not have a private IP address,
+			// then skip adding routes, because the node is in its own location.
+			if local && t.privateIP != nil {
+				for i := range segment.cidrs {
+					// Don't add routes for the local node.
+					if segment.privateIPs[i].Equal(t.privateIP.IP) {
+						continue
+					}
+					routes = append(routes, encapsulateRoute(&netlink.Route{
+						Dst:       segment.cidrs[i],
+						Flags:     int(netlink.FLAG_ONLINK),
+						Gw:        segment.privateIPs[i],
+						LinkIndex: privIface,
+						Protocol:  unix.RTPROT_STATIC,
+					}, enc.Strategy(), t.privateIP, tunlIface))
+					// Encapsulate packets from the host's Pod subnet headed
+					// to private IPs.
+					if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
+						routes = append(routes, &netlink.Route{
+							Dst:       oneAddressCIDR(segment.privateIPs[i]),
+							Flags:     int(netlink.FLAG_ONLINK),
+							Gw:        segment.privateIPs[i],
+							LinkIndex: tunlIface,
+							Protocol:  unix.RTPROT_STATIC,
+							Table:     kiloTableIndex,
+						})
+						rules = append(rules, defaultRule(&netlink.Rule{
+							Src:   t.subnet,
+							Dst:   oneAddressCIDR(segment.privateIPs[i]),
+							Table: kiloTableIndex,
+						}))
+						// Also encapsulate packets from the Kilo interface
+						// headed to private IPs.
+						rules = append(rules, defaultRule(&netlink.Rule{
+							Dst:     oneAddressCIDR(segment.privateIPs[i]),
+							Table:   kiloTableIndex,
+							IifName: kiloIfaceName,
+						}))
+					}
+				}
+			}
+			// Continuing here prevents leaders form adding routes via WireGuard to
+			// nodes in their own location.
+			continue
+		}
+		for i := range segment.cidrs {
+			// Add routes to the Pod CIDRs of nodes in other segments.
+			routes = append(routes, &netlink.Route{
+				Dst:       segment.cidrs[i],
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+			// Don't add routes through Kilo if the private IP
+			// equals the external IP. This means that the node
+			// is only accessible through an external IP and we
+			// cannot encapsulate traffic to an IP through the IP.
+			if segment.privateIPs == nil || segment.privateIPs[i].Equal(segment.endpoint.IP) {
+				continue
+			}
+			// Add routes to the private IPs of nodes in other segments.
+			// Number of CIDRs and private IPs always match so
+			// we can reuse the loop.
+			routes = append(routes, &netlink.Route{
+				Dst:       oneAddressCIDR(segment.privateIPs[i]),
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+		// For segments / locations other than the location of this instance of kg,
+		// we need to set routes for allowed location IPs over the wg interface.
+		for i := range segment.allowedLocationIPs {
+			routes = append(routes, &netlink.Route{
+				Dst:       segment.allowedLocationIPs[i],
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+	}
+	// Add routes for the allowed IPs of peers.
+	for _, peer := range t.peers {
+		for i := range peer.AllowedIPs {
+			routes = append(routes, &netlink.Route{
+				Dst:       peer.AllowedIPs[i],
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
+	}
+	return routes, rules
+}
+
+func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route {
+	if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) {
+		route.LinkIndex = tunlIface
+	}
+	return route
+}
+
+// Rules returns the iptables rules required by the local node.
+func (t *Topology) Rules(cni bool) []iptables.Rule {
+	var rules []iptables.Rule
+	rules = append(rules, iptables.NewIPv4Chain("nat", "KILO-NAT"))
+	rules = append(rules, iptables.NewIPv6Chain("nat", "KILO-NAT"))
+	if cni {
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(t.subnet.IP)), "nat", "POSTROUTING", "-s", t.subnet.String(), "-m", "comment", "--comment", "Kilo: jump to KILO-NAT chain", "-j", "KILO-NAT"))
+	}
+	for _, s := range t.segments {
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(s.wireGuardIP)), "nat", "KILO-NAT", "-d", oneAddressCIDR(s.wireGuardIP).String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuared IPs", "-j", "RETURN"))
+		for _, aip := range s.allowedIPs {
+			rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-j", "RETURN"))
+		}
+		// Make sure packets to allowed location IPs go through the KILO-NAT chain, so they can be MASQUERADEd,
+		// Otherwise packets to these destinations will reach the destination, but never find their way back.
+		// We only want to NAT in locations of the corresponding allowed location IPs.
+		if t.location == s.location {
+			for _, alip := range s.allowedLocationIPs {
+				rules = append(rules,
+					iptables.NewRule(iptables.GetProtocol(len(alip.IP)), "nat", "POSTROUTING", "-d", alip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"),
+				)
+			}
+		}
+	}
+	for _, p := range t.peers {
+		for _, aip := range p.AllowedIPs {
+			rules = append(rules,
+				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "POSTROUTING", "-s", aip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"),
+				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for peers", "-j", "RETURN"),
+			)
+		}
+	}
+	rules = append(rules, iptables.NewIPv4Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
+	rules = append(rules, iptables.NewIPv6Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
+	return rules
+}
+
+func defaultRule(rule *netlink.Rule) *netlink.Rule {
+	base := netlink.NewRule()
+	base.Src = rule.Src
+	base.Dst = rule.Dst
+	base.IifName = rule.IifName
+	base.Table = rule.Table
+	return base
+}

pkg/mesh/topology.go

@@ -19,14 +19,16 @@ import (
 	"net"
 	"sort"
 
-	"github.com/squat/kilo/pkg/encapsulation"
+	"github.com/go-kit/kit/log"
-	"github.com/squat/kilo/pkg/iptables"
+	"github.com/go-kit/kit/log/level"
+
 	"github.com/squat/kilo/pkg/wireguard"
-	"github.com/vishvananda/netlink"
-	"golang.org/x/sys/unix"
 )
 
-const kiloTableIndex = 1107
+const (
+	logicalLocationPrefix = "location:"
+	nodeLocationPrefix    = "node:"
+)
 
 // Topology represents the logical structure of the overlay network.
 type Topology struct {
@@ -51,15 +53,21 @@ type Topology struct {
 	// subnet is the Pod subnet of the local node.
 	subnet *net.IPNet
 	// wireGuardCIDR is the allocated CIDR of the WireGuard
-	// interface of the local node. If the local node is not
+	// interface of the local node within the Kilo subnet.
-	// the leader, then it is nil.
+	// If the local node is not the leader of a location, then
+	// the IP is the 0th address in the subnet, i.e. the CIDR
+	// is equal to the Kilo subnet.
 	wireGuardCIDR *net.IPNet
+	// discoveredEndpoints is the updated map of valid discovered Endpoints
+	discoveredEndpoints map[string]*wireguard.Endpoint
+	logger              log.Logger
 }
 
 type segment struct {
 	allowedIPs          []*net.IPNet
 	endpoint            *wireguard.Endpoint
 	key                 []byte
+	persistentKeepalive int
 	// Location is the logical location of this segment.
 	location string
 
@@ -74,30 +82,45 @@ type segment struct {
 	// wireGuardIP is the allocated IP address of the WireGuard
 	// interface on the leader of the segment.
 	wireGuardIP net.IP
+	// allowedLocationIPs are not part of the cluster and are not peers.
+	// They are directly routable from nodes within the segment.
+	// A classic example is a printer that ought to be routable from other locations.
+	allowedLocationIPs []*net.IPNet
 }
 
 // NewTopology creates a new Topology struct from a given set of nodes and peers.
-func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port uint32, key []byte, subnet *net.IPNet, persistentKeepalive int) (*Topology, error) {
+func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port uint32, key []byte, subnet *net.IPNet, persistentKeepalive int, logger log.Logger) (*Topology, error) {
+	if logger == nil {
+		logger = log.NewNopLogger()
+	}
 	topoMap := make(map[string][]*Node)
 	for _, node := range nodes {
 		var location string
 		switch granularity {
 		case LogicalGranularity:
-			location = node.Location
+			location = logicalLocationPrefix + node.Location
+			// Put node in a different location, if no private
+			// IP was found.
+			if node.InternalIP == nil {
+				location = nodeLocationPrefix + node.Name
+			}
 		case FullGranularity:
-			location = node.Name
+			location = nodeLocationPrefix + node.Name
 		}
 		topoMap[location] = append(topoMap[location], node)
 	}
 	var localLocation string
 	switch granularity {
 	case LogicalGranularity:
-		localLocation = nodes[hostname].Location
+		localLocation = logicalLocationPrefix + nodes[hostname].Location
+		if nodes[hostname].InternalIP == nil {
+			localLocation = nodeLocationPrefix + hostname
+		}
 	case FullGranularity:
-		localLocation = hostname
+		localLocation = nodeLocationPrefix + hostname
 	}
 
-	t := Topology{key: key, port: port, hostname: hostname, location: localLocation, persistentKeepalive: persistentKeepalive, privateIP: nodes[hostname].InternalIP, subnet: nodes[hostname].Subnet}
+	t := Topology{key: key, port: port, hostname: hostname, location: localLocation, persistentKeepalive: persistentKeepalive, privateIP: nodes[hostname].InternalIP, subnet: nodes[hostname].Subnet, wireGuardCIDR: subnet, discoveredEndpoints: make(map[string]*wireguard.Endpoint), logger: logger}
 	for location := range topoMap {
 		// Sort the location so the result is stable.
 		sort.Slice(topoMap[location], func(i, j int) bool {
@@ -108,6 +131,8 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			t.leader = true
 		}
 		var allowedIPs []*net.IPNet
+		allowedLocationIPsMap := make(map[string]struct{})
+		var allowedLocationIPs []*net.IPNet
 		var cidrs []*net.IPNet
 		var hostnames []string
 		var privateIPs []net.IP
@@ -116,20 +141,36 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			// - the node's allocated subnet
 			// - the node's WireGuard IP
 			// - the node's internal IP
-			allowedIPs = append(allowedIPs, node.Subnet, oneAddressCIDR(node.InternalIP.IP))
+			// - IPs that were specified by the allowed-location-ips annotation
-			cidrs = append(cidrs, node.Subnet)
+			allowedIPs = append(allowedIPs, node.Subnet)
-			hostnames = append(hostnames, node.Name)
+			for _, ip := range node.AllowedLocationIPs {
+				if _, ok := allowedLocationIPsMap[ip.String()]; !ok {
+					allowedLocationIPs = append(allowedLocationIPs, ip)
+					allowedLocationIPsMap[ip.String()] = struct{}{}
+				}
+			}
+			if node.InternalIP != nil {
+				allowedIPs = append(allowedIPs, oneAddressCIDR(node.InternalIP.IP))
 				privateIPs = append(privateIPs, node.InternalIP.IP)
 			}
+			cidrs = append(cidrs, node.Subnet)
+			hostnames = append(hostnames, node.Name)
+		}
+		// The sorting has no function, but makes testing easier.
+		sort.Slice(allowedLocationIPs, func(i, j int) bool {
+			return allowedLocationIPs[i].String() < allowedLocationIPs[j].String()
+		})
 		t.segments = append(t.segments, &segment{
 			allowedIPs:          allowedIPs,
 			endpoint:            topoMap[location][leader].Endpoint,
 			key:                 topoMap[location][leader].Key,
+			persistentKeepalive: topoMap[location][leader].PersistentKeepalive,
 			location:            location,
 			cidrs:               cidrs,
 			hostnames:           hostnames,
 			leader:              leader,
 			privateIPs:          privateIPs,
+			allowedLocationIPs:  allowedLocationIPs,
 		})
 	}
 	// Sort the Topology segments so the result is stable.
@@ -147,6 +188,10 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 	// We need to defensively deduplicate peer allowed IPs. If two peers claim the same IP,
 	// the WireGuard configuration could flap, causing the interface to churn.
 	t.peers = deduplicatePeerIPs(t.peers)
+	// Copy the host node DiscoveredEndpoints in the topology as a starting point.
+	for key := range nodes[hostname].DiscoveredEndpoints {
+		t.discoveredEndpoints[key] = nodes[hostname].DiscoveredEndpoints[key]
+	}
 	// Allocate IPs to the segment leaders in a stable, coordination-free manner.
 	a := newAllocator(*subnet)
 	for _, segment := range t.segments {
@@ -159,196 +204,79 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 		if t.leader && segment.location == t.location {
 			t.wireGuardCIDR = &net.IPNet{IP: ipNet.IP, Mask: subnet.Mask}
 		}
+
+		// Now that the topology is ordered, update the discoveredEndpoints map
+		// add new ones by going through the ordered topology: segments, nodes
+		for _, node := range topoMap[segment.location] {
+			for key := range node.DiscoveredEndpoints {
+				if _, ok := t.discoveredEndpoints[key]; !ok {
+					t.discoveredEndpoints[key] = node.DiscoveredEndpoints[key]
+				}
+			}
+		}
+		// Check for intersecting IPs in allowed location IPs
+		segment.allowedLocationIPs = t.filterAllowedLocationIPs(segment.allowedLocationIPs, segment.location)
 	}
 
 	return &t, nil
 }
 
-// Routes generates a slice of routes for a given Topology.
+func intersect(n1, n2 *net.IPNet) bool {
-func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface int, local bool, enc encapsulation.Encapsulator) ([]*netlink.Route, []*netlink.Rule) {
+	return n1.Contains(n2.IP) || n2.Contains(n1.IP)
-	var routes []*netlink.Route
-	var rules []*netlink.Rule
-	if !t.leader {
-		// Find the GW for this segment.
-		// This will be the an IP of the leader.
-		// In an IPIP encapsulated mesh it is the leader's private IP.
-		var gw net.IP
-		for _, segment := range t.segments {
-			if segment.location == t.location {
-				gw = enc.Gw(segment.endpoint.IP, segment.privateIPs[segment.leader], segment.cidrs[segment.leader])
-				break
-			}
-		}
-		for _, segment := range t.segments {
-			// First, add a route to the WireGuard IP of the segment.
-			routes = append(routes, encapsulateRoute(&netlink.Route{
-				Dst:       oneAddressCIDR(segment.wireGuardIP),
-				Flags:     int(netlink.FLAG_ONLINK),
-				Gw:        gw,
-				LinkIndex: privIface,
-				Protocol:  unix.RTPROT_STATIC,
-			}, enc.Strategy(), t.privateIP, tunlIface))
-			// Add routes for the current segment if local is true.
-			if segment.location == t.location {
-				if local {
-					for i := range segment.cidrs {
-						// Don't add routes for the local node.
-						if segment.privateIPs[i].Equal(t.privateIP.IP) {
-							continue
-						}
-						routes = append(routes, encapsulateRoute(&netlink.Route{
-							Dst:       segment.cidrs[i],
-							Flags:     int(netlink.FLAG_ONLINK),
-							Gw:        segment.privateIPs[i],
-							LinkIndex: privIface,
-							Protocol:  unix.RTPROT_STATIC,
-						}, enc.Strategy(), t.privateIP, tunlIface))
-						// Encapsulate packets from the host's Pod subnet headed
-						// to private IPs.
-						if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
-							routes = append(routes, &netlink.Route{
-								Dst:       oneAddressCIDR(segment.privateIPs[i]),
-								Flags:     int(netlink.FLAG_ONLINK),
-								Gw:        segment.privateIPs[i],
-								LinkIndex: tunlIface,
-								Protocol:  unix.RTPROT_STATIC,
-								Table:     kiloTableIndex,
-							})
-							rules = append(rules, defaultRule(&netlink.Rule{
-								Src:   t.subnet,
-								Dst:   oneAddressCIDR(segment.privateIPs[i]),
-								Table: kiloTableIndex,
-							}))
-						}
-					}
-				}
-				continue
-			}
-			for i := range segment.cidrs {
-				// Add routes to the Pod CIDRs of nodes in other segments.
-				routes = append(routes, encapsulateRoute(&netlink.Route{
-					Dst:       segment.cidrs[i],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        gw,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				}, enc.Strategy(), t.privateIP, tunlIface))
-				// Add routes to the private IPs of nodes in other segments.
-				// Number of CIDRs and private IPs always match so
-				// we can reuse the loop.
-				routes = append(routes, encapsulateRoute(&netlink.Route{
-					Dst:       oneAddressCIDR(segment.privateIPs[i]),
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        gw,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				}, enc.Strategy(), t.privateIP, tunlIface))
-			}
-		}
-		// Add routes for the allowed IPs of peers.
-		for _, peer := range t.peers {
-			for i := range peer.AllowedIPs {
-				routes = append(routes, encapsulateRoute(&netlink.Route{
-					Dst:       peer.AllowedIPs[i],
-					Flags:     int(netlink.FLAG_ONLINK),
-					Gw:        gw,
-					LinkIndex: privIface,
-					Protocol:  unix.RTPROT_STATIC,
-				}, enc.Strategy(), t.privateIP, tunlIface))
-			}
-		}
-		return routes, rules
-	}
-	for _, segment := range t.segments {
-		// Add routes for the current segment if local is true.
-		if segment.location == t.location {
-			if local {
-				for i := range segment.cidrs {
-					// Don't add routes for the local node.
-					if segment.privateIPs[i].Equal(t.privateIP.IP) {
-						continue
-					}
-					routes = append(routes, encapsulateRoute(&netlink.Route{
-						Dst:       segment.cidrs[i],
-						Flags:     int(netlink.FLAG_ONLINK),
-						Gw:        segment.privateIPs[i],
-						LinkIndex: privIface,
-						Protocol:  unix.RTPROT_STATIC,
-					}, enc.Strategy(), t.privateIP, tunlIface))
-					// Encapsulate packets from the host's Pod subnet headed
-					// to private IPs.
-					if enc.Strategy() == encapsulation.Always || (enc.Strategy() == encapsulation.CrossSubnet && !t.privateIP.Contains(segment.privateIPs[i])) {
-						routes = append(routes, &netlink.Route{
-							Dst:       oneAddressCIDR(segment.privateIPs[i]),
-							Flags:     int(netlink.FLAG_ONLINK),
-							Gw:        segment.privateIPs[i],
-							LinkIndex: tunlIface,
-							Protocol:  unix.RTPROT_STATIC,
-							Table:     kiloTableIndex,
-						})
-						rules = append(rules, defaultRule(&netlink.Rule{
-							Src:   t.subnet,
-							Dst:   oneAddressCIDR(segment.privateIPs[i]),
-							Table: kiloTableIndex,
-						}))
-						// Also encapsulate packets from the Kilo interface
-						// headed to private IPs.
-						rules = append(rules, defaultRule(&netlink.Rule{
-							Dst:     oneAddressCIDR(segment.privateIPs[i]),
-							Table:   kiloTableIndex,
-							IifName: kiloIfaceName,
-						}))
-					}
-				}
-			}
-			continue
-		}
-		for i := range segment.cidrs {
-			// Add routes to the Pod CIDRs of nodes in other segments.
-			routes = append(routes, &netlink.Route{
-				Dst:       segment.cidrs[i],
-				Flags:     int(netlink.FLAG_ONLINK),
-				Gw:        segment.wireGuardIP,
-				LinkIndex: kiloIface,
-				Protocol:  unix.RTPROT_STATIC,
-			})
-			// Don't add routes through Kilo if the private IP
-			// equals the external IP. This means that the node
-			// is only accessible through an external IP and we
-			// cannot encapsulate traffic to an IP through the IP.
-			if segment.privateIPs[i].Equal(segment.endpoint.IP) {
-				continue
-			}
-			// Add routes to the private IPs of nodes in other segments.
-			// Number of CIDRs and private IPs always match so
-			// we can reuse the loop.
-			routes = append(routes, &netlink.Route{
-				Dst:       oneAddressCIDR(segment.privateIPs[i]),
-				Flags:     int(netlink.FLAG_ONLINK),
-				Gw:        segment.wireGuardIP,
-				LinkIndex: kiloIface,
-				Protocol:  unix.RTPROT_STATIC,
-			})
-		}
-	}
-	// Add routes for the allowed IPs of peers.
-	for _, peer := range t.peers {
-		for i := range peer.AllowedIPs {
-			routes = append(routes, &netlink.Route{
-				Dst:       peer.AllowedIPs[i],
-				LinkIndex: kiloIface,
-				Protocol:  unix.RTPROT_STATIC,
-			})
-		}
-	}
-	return routes, rules
 }
 
-func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route {
+func (t *Topology) filterAllowedLocationIPs(ips []*net.IPNet, location string) (ret []*net.IPNet) {
-	if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) {
+CheckIPs:
-		route.LinkIndex = tunlIface
+	for _, ip := range ips {
+		for _, s := range t.segments {
+			// Check if allowed location IPs are also allowed in other locations.
+			if location != s.location {
+				for _, i := range s.allowedLocationIPs {
+					if intersect(ip, i) {
+						level.Warn(t.logger).Log("msg", "overlapping allowed location IPnets", "IP", ip.String(), "IP2", i.String(), "segment-location", s.location)
+						continue CheckIPs
 					}
-	return route
+				}
+			}
+			// Check if allowed location IPs intersect with the allowed IPs.
+			for _, i := range s.allowedIPs {
+				if intersect(ip, i) {
+					level.Warn(t.logger).Log("msg", "overlapping allowed location IPnet with allowed IPnets", "IP", ip.String(), "IP2", i.String(), "segment-location", s.location)
+					continue CheckIPs
+				}
+			}
+			// Check if allowed location IPs intersect with the private IPs of the segment.
+			for _, i := range s.privateIPs {
+				if ip.Contains(i) {
+					level.Warn(t.logger).Log("msg", "overlapping allowed location IPnet with privateIP", "IP", ip.String(), "IP2", i.String(), "segment-location", s.location)
+					continue CheckIPs
+				}
+			}
+		}
+		// Check if allowed location IPs intersect with allowed IPs of peers.
+		for _, p := range t.peers {
+			for _, i := range p.AllowedIPs {
+				if intersect(ip, i) {
+					level.Warn(t.logger).Log("msg", "overlapping allowed location IPnet with peer IPnet", "IP", ip.String(), "IP2", i.String(), "peer", p.Name)
+					continue CheckIPs
+				}
+			}
+		}
+		ret = append(ret, ip)
+	}
+	return
+}
+
+func (t *Topology) updateEndpoint(endpoint *wireguard.Endpoint, key []byte, persistentKeepalive int) *wireguard.Endpoint {
+	// Do not update non-nat peers
+	if persistentKeepalive == 0 {
+		return endpoint
+	}
+	e, ok := t.discoveredEndpoints[string(key)]
+	if ok {
+		return e
+	}
+	return endpoint
 }
 
 // Conf generates a WireGuard configuration file for a given Topology.
@@ -364,8 +292,8 @@ func (t *Topology) Conf() *wireguard.Conf {
 			continue
 		}
 		peer := &wireguard.Peer{
-			AllowedIPs:          s.allowedIPs,
+			AllowedIPs:          append(s.allowedIPs, s.allowedLocationIPs...),
-			Endpoint:            s.endpoint,
+			Endpoint:            t.updateEndpoint(s.endpoint, s.key, s.persistentKeepalive),
 			PersistentKeepalive: t.persistentKeepalive,
 			PublicKey:           s.key,
 		}
@@ -374,7 +302,7 @@ func (t *Topology) Conf() *wireguard.Conf {
 	for _, p := range t.peers {
 		peer := &wireguard.Peer{
 			AllowedIPs:          p.AllowedIPs,
-			Endpoint:            p.Endpoint,
+			Endpoint:            t.updateEndpoint(p.Endpoint, p.PublicKey, p.PersistentKeepalive),
 			PersistentKeepalive: t.persistentKeepalive,
 			PresharedKey:        p.PresharedKey,
 			PublicKey:           p.PublicKey,
@@ -437,33 +365,6 @@ func (t *Topology) PeerConf(name string) *wireguard.Conf {
 	return c
 }
 
-// Rules returns the iptables rules required by the local node.
-func (t *Topology) Rules(cni bool) []iptables.Rule {
-	var rules []iptables.Rule
-	rules = append(rules, iptables.NewIPv4Chain("nat", "KILO-NAT"))
-	rules = append(rules, iptables.NewIPv6Chain("nat", "KILO-NAT"))
-	if cni {
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(t.subnet.IP)), "nat", "POSTROUTING", "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-s", t.subnet.String(), "-j", "KILO-NAT"))
-	}
-	for _, s := range t.segments {
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(s.wireGuardIP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuared IPs", "-d", s.wireGuardIP.String(), "-j", "RETURN"))
-		for _, aip := range s.allowedIPs {
-			rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-d", aip.String(), "-j", "RETURN"))
-		}
-	}
-	for _, p := range t.peers {
-		for _, aip := range p.AllowedIPs {
-			rules = append(rules,
-				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "POSTROUTING", "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-s", aip.String(), "-j", "KILO-NAT"),
-				iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: do not NAT packets destined for peers", "-d", aip.String(), "-j", "RETURN"),
-			)
-		}
-	}
-	rules = append(rules, iptables.NewIPv4Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
-	rules = append(rules, iptables.NewIPv6Rule("nat", "KILO-NAT", "-m", "comment", "--comment", "Kilo: NAT remaining packets", "-j", "MASQUERADE"))
-	return rules
-}
-
 // oneAddressCIDR takes an IP address and returns a CIDR
 // that contains only that address.
 func oneAddressCIDR(ip net.IP) *net.IPNet {
@@ -521,12 +422,3 @@ func deduplicatePeerIPs(peers []*Peer) []*Peer {
 	}
 	return ps
 }
-
-func defaultRule(rule *netlink.Rule) *netlink.Rule {
-	base := netlink.NewRule()
-	base.Src = rule.Src
-	base.Dst = rule.Dst
-	base.IifName = rule.IifName
-	base.Table = rule.Table
-	return base
-}

pkg/route/route.go

@@ -108,14 +108,14 @@ func (t *Table) Run(stop <-chan struct{}) (<-chan error, error) {
 			switch e.Type {
 			// Watch for deleted routes to reconcile this table's routes.
 			case unix.RTM_DELROUTE:
+				// Filter out invalid routes.
+				if e.Route.Dst == nil {
+					continue
+				}
 				t.mu.Lock()
 				for k := range t.rs {
 					switch r := t.rs[k].(type) {
 					case *netlink.Route:
-						// Filter out invalid routes.
-						if r == nil || r.Dst == nil {
-							continue
-						}
 						// If any deleted route's destination matches a destination
 						// in the table, reset the corresponding route just in case.
 						if r.Dst.IP.Equal(e.Route.Dst.IP) && r.Dst.Mask.String() == e.Route.Dst.Mask.String() {

pkg/wireguard/conf.go

@@ -17,11 +17,13 @@ package wireguard
 import (
 	"bufio"
 	"bytes"
+	"errors"
 	"fmt"
 	"net"
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"k8s.io/apimachinery/pkg/util/validation"
 )
@@ -31,6 +33,9 @@ type key string
 
 const (
 	separator                      = "="
+	dumpSeparator                  = "\t"
+	dumpNone                       = "(none)"
+	dumpOff                        = "off"
 	interfaceSection       section = "Interface"
 	peerSection            section = "Peer"
 	listenPortKey          key     = "ListenPort"
@@ -42,6 +47,30 @@ const (
 	publicKeyKey           key     = "PublicKey"
 )
 
+type dumpInterfaceIndex int
+
+const (
+	dumpInterfacePrivateKeyIndex = iota
+	dumpInterfacePublicKeyIndex
+	dumpInterfaceListenPortIndex
+	dumpInterfaceFWMarkIndex
+	dumpInterfaceLen
+)
+
+type dumpPeerIndex int
+
+const (
+	dumpPeerPublicKeyIndex = iota
+	dumpPeerPresharedKeyIndex
+	dumpPeerEndpointIndex
+	dumpPeerAllowedIPsIndex
+	dumpPeerLatestHandshakeIndex
+	dumpPeerTransferRXIndex
+	dumpPeerTransferTXIndex
+	dumpPeerPersistentKeepaliveIndex
+	dumpPeerLen
+)
+
 // Conf represents a WireGuard configuration file.
 type Conf struct {
 	Interface *Interface
@@ -61,6 +90,8 @@ type Peer struct {
 	PersistentKeepalive int
 	PresharedKey        []byte
 	PublicKey           []byte
+	// The following fields are part of the runtime information, not the configuration.
+	LatestHandshake time.Time
 }
 
 // DeduplicateIPs eliminates duplicate allowed IPs.
@@ -95,6 +126,38 @@ func (e *Endpoint) String() string {
 	return dnsOrIP + ":" + strconv.FormatUint(uint64(e.Port), 10)
 }
 
+// Equal compares two endpoints.
+func (e *Endpoint) Equal(b *Endpoint, DNSFirst bool) bool {
+	if (e == nil) != (b == nil) {
+		return false
+	}
+	if e != nil {
+		if e.Port != b.Port {
+			return false
+		}
+		if DNSFirst {
+			// Check the DNS name first if it was resolved.
+			if e.DNS != b.DNS {
+				return false
+			}
+			if e.DNS == "" && !e.IP.Equal(b.IP) {
+				return false
+			}
+		} else {
+			// IPs take priority, so check them first.
+			if !e.IP.Equal(b.IP) {
+				return false
+			}
+			// Only check the DNS name if the IP is empty.
+			if e.IP == nil && e.DNS != b.DNS {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
 // DNSOrIP represents either a DNS name or an IP address.
 // IPs, as they are more specific, are preferred.
 type DNSOrIP struct {
@@ -114,13 +177,11 @@ func (d DNSOrIP) String() string {
 func Parse(buf []byte) *Conf {
 	var (
 		active  section
-		ai      *net.IPNet
 		kv      []string
 		c       Conf
 		err     error
 		iface   *Interface
 		i       int
-		ip, ip4 net.IP
 		k       key
 		line, v string
 		peer    *Peer
@@ -173,49 +234,15 @@ func Parse(buf []byte) *Conf {
 		case peerSection:
 			switch k {
 			case allowedIPsKey:
-				// Reuse string slice.
+				err = peer.parseAllowedIPs(v)
-				kv = strings.Split(v, ",")
-				for i = range kv {
-					ip, ai, err = net.ParseCIDR(strings.TrimSpace(kv[i]))
 				if err != nil {
 					continue
 				}
-					if ip4 = ip.To4(); ip4 != nil {
-						ip = ip4
-					} else {
-						ip = ip.To16()
-					}
-					ai.IP = ip
-					peer.AllowedIPs = append(peer.AllowedIPs, ai)
-				}
 			case endpointKey:
-				// Reuse string slice.
+				err = peer.parseEndpoint(v)
-				kv = strings.Split(v, ":")
-				if len(kv) < 2 {
-					continue
-				}
-				port, err = strconv.ParseUint(kv[len(kv)-1], 10, 32)
 				if err != nil {
 					continue
 				}
-				d := DNSOrIP{}
-				ip = net.ParseIP(strings.Trim(strings.Join(kv[:len(kv)-1], ":"), "[]"))
-				if ip == nil {
-					if len(validation.IsDNS1123Subdomain(kv[0])) != 0 {
-						continue
-					}
-					d.DNS = kv[0]
-				} else {
-					if ip4 = ip.To4(); ip4 != nil {
-						d.IP = ip4
-					} else {
-						d.IP = ip.To16()
-					}
-				}
-				peer.Endpoint = &Endpoint{
-					DNSOrIP: d,
-					Port:    uint32(port),
-				}
 			case persistentKeepaliveKey:
 				i, err = strconv.Atoi(v)
 				if err != nil {
@@ -309,22 +336,9 @@ func (c *Conf) Equal(b *Conf) bool {
 				return false
 			}
 		}
-		if (c.Peers[i].Endpoint == nil) != (b.Peers[i].Endpoint == nil) {
+		if !c.Peers[i].Endpoint.Equal(b.Peers[i].Endpoint, false) {
 			return false
 		}
-		if c.Peers[i].Endpoint != nil {
-			if c.Peers[i].Endpoint.Port != b.Peers[i].Endpoint.Port {
-				return false
-			}
-			// IPs take priority, so check them first.
-			if !c.Peers[i].Endpoint.IP.Equal(b.Peers[i].Endpoint.IP) {
-				return false
-			}
-			// Only check the DNS name if the IP is empty.
-			if c.Peers[i].Endpoint.IP == nil && c.Peers[i].Endpoint.DNS != b.Peers[i].Endpoint.DNS {
-				return false
-			}
-		}
 		if c.Peers[i].PersistentKeepalive != b.Peers[i].PersistentKeepalive || !bytes.Equal(c.Peers[i].PresharedKey, b.Peers[i].PresharedKey) || !bytes.Equal(c.Peers[i].PublicKey, b.Peers[i].PublicKey) {
 			return false
 		}
@@ -429,3 +443,177 @@ func writeKey(buf *bytes.Buffer, k key) error {
 	_, err = buf.WriteString(" = ")
 	return err
 }
+
+var (
+	errParseEndpoint = errors.New("could not parse Endpoint")
+)
+
+func (p *Peer) parseEndpoint(v string) error {
+	var (
+		kv      []string
+		err     error
+		ip, ip4 net.IP
+		port    uint64
+	)
+	kv = strings.Split(v, ":")
+	if len(kv) < 2 {
+		return errParseEndpoint
+	}
+	port, err = strconv.ParseUint(kv[len(kv)-1], 10, 32)
+	if err != nil {
+		return err
+	}
+	d := DNSOrIP{}
+	ip = net.ParseIP(strings.Trim(strings.Join(kv[:len(kv)-1], ":"), "[]"))
+	if ip == nil {
+		if len(validation.IsDNS1123Subdomain(kv[0])) != 0 {
+			return errParseEndpoint
+		}
+		d.DNS = kv[0]
+	} else {
+		if ip4 = ip.To4(); ip4 != nil {
+			d.IP = ip4
+		} else {
+			d.IP = ip.To16()
+		}
+	}
+
+	p.Endpoint = &Endpoint{
+		DNSOrIP: d,
+		Port:    uint32(port),
+	}
+	return nil
+}
+
+func (p *Peer) parseAllowedIPs(v string) error {
+	var (
+		ai      *net.IPNet
+		kv      []string
+		err     error
+		i       int
+		ip, ip4 net.IP
+	)
+
+	kv = strings.Split(v, ",")
+	for i = range kv {
+		ip, ai, err = net.ParseCIDR(strings.TrimSpace(kv[i]))
+		if err != nil {
+			return err
+		}
+		if ip4 = ip.To4(); ip4 != nil {
+			ip = ip4
+		} else {
+			ip = ip.To16()
+		}
+		ai.IP = ip
+		p.AllowedIPs = append(p.AllowedIPs, ai)
+	}
+	return nil
+}
+
+// ParseDump parses a given WireGuard dump and produces a Conf struct.
+func ParseDump(buf []byte) (*Conf, error) {
+	// from man wg, show section:
+	// If dump is specified, then several lines are printed;
+	// the first contains in order separated by tab: private-key, public-key, listen-port, fw‐mark.
+	// Subsequent lines are printed for each peer and contain in order separated by tab:
+	// public-key, preshared-key, endpoint, allowed-ips, latest-handshake, transfer-rx, transfer-tx, persistent-keepalive.
+	var (
+		active section
+		values []string
+		c      Conf
+		err    error
+		iface  *Interface
+		peer   *Peer
+		port   uint64
+		sec    int64
+		pka    int
+		line   int
+	)
+	// First line is Interface
+	active = interfaceSection
+	s := bufio.NewScanner(bytes.NewBuffer(buf))
+	for s.Scan() {
+		values = strings.Split(s.Text(), dumpSeparator)
+
+		switch active {
+		case interfaceSection:
+			if len(values) < dumpInterfaceLen {
+				return nil, fmt.Errorf("invalid interface line: missing fields (%d < %d)", len(values), dumpInterfaceLen)
+			}
+			iface = new(Interface)
+			for i := range values {
+				switch i {
+				case dumpInterfacePrivateKeyIndex:
+					iface.PrivateKey = []byte(values[i])
+				case dumpInterfaceListenPortIndex:
+					port, err = strconv.ParseUint(values[i], 10, 32)
+					if err != nil {
+						return nil, fmt.Errorf("invalid interface line: error parsing listen-port: %w", err)
+					}
+					iface.ListenPort = uint32(port)
+				}
+			}
+			c.Interface = iface
+			// Next lines are Peers
+			active = peerSection
+		case peerSection:
+			if len(values) < dumpPeerLen {
+				return nil, fmt.Errorf("invalid peer line %d: missing fields (%d < %d)", line, len(values), dumpPeerLen)
+			}
+			peer = new(Peer)
+
+			for i := range values {
+				switch i {
+				case dumpPeerPublicKeyIndex:
+					peer.PublicKey = []byte(values[i])
+				case dumpPeerPresharedKeyIndex:
+					if values[i] == dumpNone {
+						continue
+					}
+					peer.PresharedKey = []byte(values[i])
+				case dumpPeerEndpointIndex:
+					if values[i] == dumpNone {
+						continue
+					}
+					err = peer.parseEndpoint(values[i])
+					if err != nil {
+						return nil, fmt.Errorf("invalid peer line %d: error parsing endpoint: %w", line, err)
+					}
+				case dumpPeerAllowedIPsIndex:
+					if values[i] == dumpNone {
+						continue
+					}
+					err = peer.parseAllowedIPs(values[i])
+					if err != nil {
+						return nil, fmt.Errorf("invalid peer line %d: error parsing allowed-ips: %w", line, err)
+					}
+				case dumpPeerLatestHandshakeIndex:
+					if values[i] == "0" {
+						// Use go zero value, not unix 0 timestamp.
+						peer.LatestHandshake = time.Time{}
+						continue
+					}
+					sec, err = strconv.ParseInt(values[i], 10, 64)
+					if err != nil {
+						return nil, fmt.Errorf("invalid peer line %d: error parsing latest-handshake: %w", line, err)
+					}
+					peer.LatestHandshake = time.Unix(sec, 0)
+				case dumpPeerPersistentKeepaliveIndex:
+					if values[i] == dumpOff {
+						continue
+					}
+					pka, err = strconv.Atoi(values[i])
+					if err != nil {
+						return nil, fmt.Errorf("invalid peer line %d: error parsing persistent-keepalive: %w", line, err)
+					}
+					peer.PersistentKeepalive = pka
+				}
+			}
+			c.Peers = append(c.Peers, peer)
+			peer = nil
+		}
+		line++
+	}
+	return &c, nil
+}

pkg/wireguard/conf_test.go

@@ -15,7 +15,10 @@
 package wireguard
 
 import (
+	"net"
 	"testing"
+
+	"github.com/kylelemons/godebug/pretty"
 )
 
 func TestCompareConf(t *testing.T) {
@@ -203,3 +206,151 @@ func TestCompareConf(t *testing.T) {
 		}
 	}
 }
+
+func TestCompareEndpoint(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		a        *Endpoint
+		b        *Endpoint
+		dnsFirst bool
+		out      bool
+	}{
+		{
+			name: "both nil",
+			a:    nil,
+			b:    nil,
+			out:  true,
+		},
+		{
+			name: "a nil",
+			a:    nil,
+			b:    &Endpoint{},
+			out:  false,
+		},
+		{
+			name: "b nil",
+			a:    &Endpoint{},
+			b:    nil,
+			out:  false,
+		},
+		{
+			name: "zero",
+			a:    &Endpoint{},
+			b:    &Endpoint{},
+			out:  true,
+		},
+		{
+			name: "diff port",
+			a:    &Endpoint{Port: 1234},
+			b:    &Endpoint{Port: 5678},
+			out:  false,
+		},
+		{
+			name: "same IP",
+			a:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1")}},
+			b:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1")}},
+			out:  true,
+		},
+		{
+			name: "diff IP",
+			a:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1")}},
+			b:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.2")}},
+			out:  false,
+		},
+		{
+			name: "same IP ignore DNS",
+			a:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1"), DNS: "a"}},
+			b:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1"), DNS: "b"}},
+			out:  true,
+		},
+		{
+			name: "no IP check DNS",
+			a:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{DNS: "a"}},
+			b:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{DNS: "b"}},
+			out:  false,
+		},
+		{
+			name: "no IP check DNS (same)",
+			a:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{DNS: "a"}},
+			b:    &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{DNS: "a"}},
+			out:  true,
+		},
+		{
+			name:     "DNS first, ignore IP",
+			a:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1"), DNS: "a"}},
+			b:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.2"), DNS: "a"}},
+			dnsFirst: true,
+			out:      true,
+		},
+		{
+			name:     "DNS first",
+			a:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{DNS: "a"}},
+			b:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{DNS: "b"}},
+			dnsFirst: true,
+			out:      false,
+		},
+		{
+			name:     "DNS first, no DNS compare IP",
+			a:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1"), DNS: ""}},
+			b:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.2"), DNS: ""}},
+			dnsFirst: true,
+			out:      false,
+		},
+		{
+			name:     "DNS first, no DNS compare IP (same)",
+			a:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1"), DNS: ""}},
+			b:        &Endpoint{Port: 1234, DNSOrIP: DNSOrIP{IP: net.ParseIP("192.168.0.1"), DNS: ""}},
+			dnsFirst: true,
+			out:      true,
+		},
+	} {
+		equal := tc.a.Equal(tc.b, tc.dnsFirst)
+		if equal != tc.out {
+			t.Errorf("test case %q: expected %t, got %t", tc.name, tc.out, equal)
+		}
+	}
+}
+
+func TestCompareDumpConf(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		d    []byte
+		c    []byte
+	}{
+		{
+			name: "empty",
+			d:    []byte{},
+			c:    []byte{},
+		},
+		{
+			name: "redacted copy from wg output",
+			d: []byte(`private	B7qk8EMlob0nfado0ABM6HulUV607r4yqtBKjhap7S4=	51820	off
+key1	(none)	10.254.1.1:51820	100.64.1.0/24,192.168.0.125/32,10.4.0.1/32	1619012801	67048	34952	10
+key2	(none)	10.254.2.1:51820	100.64.4.0/24,10.69.76.55/32,100.64.3.0/24,10.66.25.131/32,10.4.0.2/32	1619013058	1134456	10077852	10`),
+			c: []byte(`[Interface]
+		ListenPort = 51820
+		PrivateKey = private
+
+		[Peer]
+		PublicKey = key1
+		AllowedIPs = 100.64.1.0/24, 192.168.0.125/32, 10.4.0.1/32
+		Endpoint = 10.254.1.1:51820
+		PersistentKeepalive = 10
+
+		[Peer]
+		PublicKey = key2
+		AllowedIPs = 100.64.4.0/24, 10.69.76.55/32, 100.64.3.0/24, 10.66.25.131/32, 10.4.0.2/32
+		Endpoint = 10.254.2.1:51820
+		PersistentKeepalive = 10`),
+		},
+	} {
+
+		dumpConf, _ := ParseDump(tc.d)
+		conf := Parse(tc.c)
+		// Equal will ignore runtime fields and only compare configuration fields.
+		if !dumpConf.Equal(conf) {
+			diff := pretty.Compare(dumpConf, conf)
+			t.Errorf("test case %q: got diff: %v", tc.name, diff)
+		}
+	}
+}