manifests/: fix boringtun containers

A change in boringtun's cli caused the boringtun containers to crash.

Signed-off-by: leonnicolas <leonloechner@gmx.de>

.dockerignore

@@ -0,0 +1,3 @@
+**
+
+!/bin/linux

.github/workflows/ci.yml

@@ -0,0 +1,183 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+    tags:
+      - "*"
+  pull_request:
+  schedule:
+  - cron:  '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+
+  vendor:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Vendor
+      run: |
+        make vendor
+        git diff --exit-code
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Build
+      run: make
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Build docs
+      run: |
+        make gen-docs
+        git diff --exit-code
+
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Build kg and kgctl for all Linux Architectures
+      run: make all-build
+
+  darwin:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Build kgctl for Darwin amd64
+      run: make OS=darwin ARCH=amd64
+    - name: Build kgctl for Darwin arm64
+      run: make OS=darwin ARCH=arm64
+
+  windows:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Build kgctl for Windows
+      run: make OS=windows
+
+  unit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Run Unit Tests
+      run: make unit
+
+  e2e:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Run e2e Tests
+      run: make e2e
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Lint Code
+      run: make lint
+
+  container:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Enable Experimental Docker CLI
+      run: |
+        echo $'{\n  "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+        mkdir -p ~/.docker
+        echo $'{\n  "experimental": "enabled"\n}' | sudo tee ~/.docker/config.json
+        sudo service docker restart
+        docker version -f '{{.Client.Experimental}}'
+        docker version -f '{{.Server.Experimental}}'
+        docker buildx version
+    - name: Container
+      run: make container
+
+  push:
+    if: github.event_name != 'pull_request'
+    needs:
+    - vendor
+    - build
+    - linux
+    - darwin
+    - windows
+    - unit
+    - lint
+    - container
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Enable Experimental Docker CLI
+      run: |
+        echo $'{\n  "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+        mkdir -p ~/.docker
+        echo $'{\n  "experimental": "enabled"\n}' | sudo tee ~/.docker/config.json
+        sudo service docker restart
+        docker version -f '{{.Client.Experimental}}'
+        docker version -f '{{.Server.Experimental}}'
+        docker buildx version
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@master
+      with:
+        platforms: all
+    - name: Login to DockerHub
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Build and push
+      if: github.event_name != 'pull_request'
+      run: make manifest
+    - name: Build and push latest
+      if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+      run: make manifest-latest

.github/workflows/release.yaml

@@ -0,0 +1,21 @@
+on:
+  release:
+    types: [created]
+name: Handle Release
+jobs:
+  kgctl:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+    - name: Build kgctl Binaries to Be Released
+      run: make release
+    - name: Publish Release
+      uses: skx/github-action-publish-binaries@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        args: 'bin/release/kgctl-*'

.gitignore

@@ -3,3 +3,5 @@
 .manifest*
 .push*
 bin/
+tmp/
+e2e/kind.yaml*

.travis.yml

@@ -1,31 +0,0 @@
-sudo: required
-
-dist: xenial
-
-language: go
-
-services:
-  - docker
-
-go:
-  - 1.14.2
-
-env:
-  - GO111MODULE=on DOCKER_CLI_EXPERIMENTAL=enabled
-
-before_install:
-  - sudo apt-get update && sudo apt-get -y install jq
-
-install: true
-
-script:
-  - make
-  - make clean
-  - make unit
-  - make lint
-  - make container
-
-after_success:
-  - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-  - make manifest && make manifest-latest

Dockerfile

@@ -1,15 +1,19 @@
 ARG FROM=alpine
-FROM alpine AS cni
+FROM $FROM AS cni
-ARG GOARCH
+ARG GOARCH=amd64
+ARG CNI_PLUGINS_VERSION=v1.1.1
 RUN apk add --no-cache curl && \
-    curl -Lo cni.tar.gz https://github.com/containernetworking/plugins/releases/download/v0.7.5/cni-plugins-$GOARCH-v0.7.5.tgz && \
+    curl -Lo cni.tar.gz https://github.com/containernetworking/plugins/releases/download/$CNI_PLUGINS_VERSION/cni-plugins-linux-$GOARCH-$CNI_PLUGINS_VERSION.tgz && \
     tar -xf cni.tar.gz
 
 FROM $FROM
 ARG GOARCH
+ARG ALPINE_VERSION=v3.12
 LABEL maintainer="squat <lserven@gmail.com>"
-RUN echo -e "https://alpine.global.ssl.fastly.net/alpine/v3.12/main\nhttps://alpine.global.ssl.fastly.net/alpine/v3.12/community" > /etc/apk/repositories && \
+RUN echo -e "https://alpine.global.ssl.fastly.net/alpine/$ALPINE_VERSION/main\nhttps://alpine.global.ssl.fastly.net/alpine/$ALPINE_VERSION/community" > /etc/apk/repositories && \
-    apk add --no-cache ipset iptables ip6tables wireguard-tools
+    apk add --no-cache ipset iptables ip6tables graphviz font-noto
 COPY --from=cni bridge host-local loopback portmap /opt/cni/bin/
-COPY bin/$GOARCH/kg /opt/bin/
+ADD https://raw.githubusercontent.com/kubernetes-sigs/iptables-wrappers/e139a115350974aac8a82ec4b815d2845f86997e/iptables-wrapper-installer.sh /
+RUN chmod 700 /iptables-wrapper-installer.sh && /iptables-wrapper-installer.sh --no-sanity-check
+COPY bin/linux/$GOARCH/kg /opt/bin/
 ENTRYPOINT ["/opt/bin/kg"]

MAINTAINERS.md

@@ -0,0 +1,6 @@
+# Core Maintainers of This Repository
+
+| Name               | GitHub                                         |
+|--------------------|------------------------------------------------|
+| Lucas Servén Marín | [@squat](https://github.com/squat)             |
+| Leon Löchner       | [@leonnicolas](https://github.com/leonnicolas) |

Makefile

@@ -1,14 +1,21 @@
 export GO111MODULE=on
-.PHONY: push container clean container-name container-latest push-latest fmt lint test unit vendor header generate client deepcopy informer lister openapi manifest manfest-latest manifest-annotate manifest manfest-latest manifest-annotate
+.PHONY: push container clean container-name container-latest push-latest fmt lint test unit vendor header generate crd client deepcopy informer lister manifest manfest-latest manifest-annotate manifest manfest-latest manifest-annotate release gen-docs e2e
 
-ARCH ?= amd64
+OS ?= $(shell go env GOOS)
+ARCH ?= $(shell go env GOARCH)
 ALL_ARCH := amd64 arm arm64
 DOCKER_ARCH := "amd64" "arm v7" "arm64 v8"
-BINS := $(addprefix bin/$(ARCH)/,kg kgctl)
+ifeq ($(OS),linux)
+    BINS := bin/$(OS)/$(ARCH)/kg bin/$(OS)/$(ARCH)/kgctl
+else
+    BINS := bin/$(OS)/$(ARCH)/kgctl
+endif
+RELEASE_BINS := $(addprefix bin/release/kgctl-, $(addprefix linux-, $(ALL_ARCH)) darwin-amd64 darwin-arm64 windows-amd64)
 PROJECT := kilo
 PKG := github.com/squat/$(PROJECT)
 REGISTRY ?= index.docker.io
 IMAGE ?= squat/$(PROJECT)
+FULLY_QUALIFIED_IMAGE := $(REGISTRY)/$(IMAGE)
 
 TAG := $(shell git describe --abbrev=0 --tags HEAD 2>/dev/null)
 COMMIT := $(shell git rev-parse HEAD)
@@ -25,20 +32,26 @@ SRC := $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 GO_FILES ?= $$(find . -name '*.go' -not -path './vendor/*')
 GO_PKGS ?= $$(go list ./... | grep -v "$(PKG)/vendor")
 
+CONTROLLER_GEN_BINARY := bin/controller-gen
 CLIENT_GEN_BINARY := bin/client-gen
+DOCS_GEN_BINARY := bin/docs-gen
 DEEPCOPY_GEN_BINARY := bin/deepcopy-gen
 INFORMER_GEN_BINARY := bin/informer-gen
 LISTER_GEN_BINARY := bin/lister-gen
-OPENAPI_GEN_BINARY := bin/openapi-gen
+STATICCHECK_BINARY := bin/staticcheck
-GOLINT_BINARY := bin/golint
+EMBEDMD_BINARY := bin/embedmd
+KIND_BINARY := $(shell pwd)/bin/kind
+KUBECTL_BINARY := $(shell pwd)/bin/kubectl
+BASH_UNIT := $(shell pwd)/bin/bash_unit
+BASH_UNIT_FLAGS :=
 
-BUILD_IMAGE ?= golang:1.14.2-alpine
+BUILD_IMAGE ?= golang:1.18.0
-BASE_IMAGE ?= alpine:3.12
+BASE_IMAGE ?= alpine:3.15
 
 build: $(BINS)
 
 build-%:
-	@$(MAKE) --no-print-directory ARCH=$* build
+	@$(MAKE) --no-print-directory OS=$(word 1,$(subst -, ,$*)) ARCH=$(word 2,$(subst -, ,$*)) build
 
 container-latest-%:
 	@$(MAKE) --no-print-directory ARCH=$* container-latest
@@ -52,7 +65,7 @@ push-latest-%:
 push-%:
 	@$(MAKE) --no-print-directory ARCH=$* push
 
-all-build: $(addprefix build-, $(ALL_ARCH))
+all-build: $(addprefix build-$(OS)-, $(ALL_ARCH))
 
 all-container: $(addprefix container-, $(ALL_ARCH))
 
@@ -62,7 +75,13 @@ all-container-latest: $(addprefix container-latest-, $(ALL_ARCH))
 
 all-push-latest: $(addprefix push-latest-, $(ALL_ARCH))
 
-generate: client deepcopy informer lister openapi
+generate: client deepcopy informer lister crd
+
+crd: manifests/crds.yaml
+manifests/crds.yaml: pkg/k8s/apis/kilo/v1alpha1/types.go $(CONTROLLER_GEN_BINARY)
+	$(CONTROLLER_GEN_BINARY) crd \
+	paths=./pkg/k8s/apis/kilo/... \
+	output:crd:stdout > $@
 
 client: pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go
 pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go: .header pkg/k8s/apis/kilo/v1alpha1/types.go $(CLIENT_GEN_BINARY)
@@ -120,19 +139,12 @@ pkg/k8s/listers/kilo/v1alpha1/peer.go: .header pkg/k8s/apis/kilo/v1alpha1/types.
 	rm -r github.com || true
 	go fmt ./pkg/k8s/listers/...
 
-openapi: pkg/k8s/apis/kilo/v1alpha1/openapi_generated.go
+gen-docs: generate docs/api.md docs/kg.md
-pkg/k8s/apis/kilo/v1alpha1/openapi_generated.go: pkg/k8s/apis/kilo/v1alpha1/types.go $(OPENAPI_GEN_BINARY)
+docs/api.md: pkg/k8s/apis/kilo/v1alpha1/types.go $(DOCS_GEN_BINARY)
-	$(OPENAPI_GEN_BINARY) \
+	$(DOCS_GEN_BINARY) $< > $@
-	--input-dirs $(PKG)/$(@D),k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/core/v1 \
-	--output-base $(CURDIR) \
-	--output-package ./$(@D) \
-	--logtostderr \
-	--report-filename /dev/null \
-	--go-header-file=.header
-	go fmt $@
 
 $(BINS): $(SRC) go.mod
-	@mkdir -p bin/$(ARCH)
+	@mkdir -p bin/$(word 2,$(subst /, ,$@))/$(word 3,$(subst /, ,$@))
 	@echo "building: $@"
 	@docker run --rm \
 	    -u $$(id -u):$$(id -g) \
@@ -140,8 +152,8 @@ $(BINS): $(SRC) go.mod
 	    -w /$(PROJECT) \
 	    $(BUILD_IMAGE) \
 	    /bin/sh -c " \
-	        GOARCH=$(ARCH) \
+	        GOARCH=$(word 3,$(subst /, ,$@)) \
-	        GOOS=linux \
+	        GOOS=$(word 2,$(subst /, ,$@)) \
 	        GOCACHE=/$(PROJECT)/.cache \
 		CGO_ENABLED=0 \
 		go build -mod=vendor -o $@ \
@@ -153,7 +165,7 @@ fmt:
 	@echo $(GO_PKGS)
 	gofmt -w -s $(GO_FILES)
 
-lint: header $(GOLINT_BINARY)
+lint: header $(STATICCHECK_BINARY)
 	@echo 'go vet $(GO_PKGS)'
 	@vet_res=$$(GO111MODULE=on go vet -mod=vendor $(GO_PKGS) 2>&1); if [ -n "$$vet_res" ]; then \
 		echo ""; \
@@ -162,10 +174,10 @@ lint: header $(GOLINT_BINARY)
 		echo "$$vet_res"; \
 		exit 1; \
 	fi
-	@echo '$(GOLINT_BINARY) $(GO_PKGS)'
+	@echo '$(STATICCHECK_BINARY) $(GO_PKGS)'
-	@lint_res=$$($(GOLINT_BINARY) $(GO_PKGS)); if [ -n "$$lint_res" ]; then \
+	@lint_res=$$($(STATICCHECK_BINARY) $(GO_PKGS)); if [ -n "$$lint_res" ]; then \
 		echo ""; \
-		echo "Golint found style issues. Please check the reported issues"; \
+		echo "Staticcheck found style issues. Please check the reported issues"; \
 		echo "and fix them if necessary before submitting the code for review:"; \
 		echo "$$lint_res"; \
 		exit 1; \
@@ -182,7 +194,22 @@ lint: header $(GOLINT_BINARY)
 unit:
 	go test -mod=vendor --race ./...
 
-test: lint unit
+test: lint unit e2e
+
+$(KIND_BINARY):
+	curl -Lo $@ https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-$(ARCH)
+	chmod +x $@
+
+$(KUBECTL_BINARY):
+	curl -Lo $@ https://dl.k8s.io/release/v1.21.0/bin/linux/$(ARCH)/kubectl
+	chmod +x $@
+
+$(BASH_UNIT):
+	curl -Lo $@ https://raw.githubusercontent.com/pgrange/bash_unit/v1.7.2/bash_unit
+	chmod +x $@
+
+e2e: container $(KIND_BINARY) $(KUBECTL_BINARY) $(BASH_UNIT) bin/$(OS)/$(ARCH)/kgctl
+	KILO_IMAGE=$(IMAGE):$(ARCH)-$(VERSION) KIND_BINARY=$(KIND_BINARY) KUBECTL_BINARY=$(KUBECTL_BINARY) KGCTL_BINARY=$(shell pwd)/bin/$(OS)/$(ARCH)/kgctl $(BASH_UNIT) $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/multi-cluster.sh ./e2e/handlers.sh ./e2e/kgctl.sh ./e2e/teardown.sh
 
 header: .header
 	@HEADER=$$(cat .header); \
@@ -190,7 +217,7 @@ header: .header
 	FILES=; \
 	for f in $(GO_FILES); do \
 		for i in 0 1 2 3 4 5; do \
-			FILE=$$(tail -n +$$i $$f | head -n $$HEADER_LEN | sed "s/[0-9]\{4\}/YEAR/"); \
+			FILE=$$(t=$$(mktemp) && tail -n +$$i $$f > $$t && head -n $$HEADER_LEN $$t | sed "s/[0-9]\{4\}/YEAR/"); \
 			[ "$$FILE" = "$$HEADER" ] && continue 2; \
 		done; \
 		FILES="$$FILES$$f "; \
@@ -200,6 +227,13 @@ header: .header
 		exit 1; \
 	fi
 
+tmp/help.txt: bin/$(OS)/$(ARCH)/kg
+	mkdir -p tmp
+	bin//$(OS)/$(ARCH)/kg --help 2>&1 | head -n -1 > $@
+
+docs/kg.md: $(EMBEDMD_BINARY) tmp/help.txt
+	$(EMBEDMD_BINARY) -w $@
+
 website/docs/README.md: README.md
 	rm -rf website/static/img/graphs
 	find docs  -type f -name '*.md' | xargs -I{} sh -c 'cat $(@D)/$$(basename {} .md) > website/{}'
@@ -208,15 +242,17 @@ website/docs/README.md: README.md
 	cat README.md >> $@
 	cp -r docs/graphs website/static/img/
 	sed -i 's/\.\/docs\///g' $@
-	find $(@D)  -type f -name '*.md' | xargs -I{} sed -i 's/\.\/\(.\+\.svg\)/\/img\/\1/g' {}
+	find $(@D)  -type f -name '*.md' | xargs -I{} sed -i 's/\.\/\(.\+\.\(svg\|png\)\)/\/img\/\1/g' {}
 	sed -i 's/graphs\//\/img\/graphs\//g' $@
+	# The next line is a workaround until mdx, docusaurus' markdown parser, can parse links with preceding brackets.
+	sed -i  's/\[\]\(\[.*\](.*)\)/\&#91;\&#93;\1/g' website/docs/api.md
 
-website/build/index.html: website/docs/README.md
+website/build/index.html: website/docs/README.md docs/api.md
 	yarn --cwd website install
 	yarn --cwd website build
 
 container: .container-$(ARCH)-$(VERSION) container-name
-.container-$(ARCH)-$(VERSION): $(BINS) Dockerfile
+.container-$(ARCH)-$(VERSION): bin/linux/$(ARCH)/kg Dockerfile
 	@i=0; for a in $(ALL_ARCH); do [ "$$a" = $(ARCH) ] && break; i=$$((i+1)); done; \
 	ia=""; iv=""; \
 	j=0; for a in $(DOCKER_ARCH); do \
@@ -227,7 +263,7 @@ container: .container-$(ARCH)-$(VERSION) container-name
 	@docker images -q $(IMAGE):$(ARCH)-$(VERSION) > $@
 
 container-latest: .container-$(ARCH)-$(VERSION)
-	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(IMAGE):$(ARCH)-latest
+	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(FULLY_QUALIFIED_IMAGE):$(ARCH)-latest
 	@echo "container: $(IMAGE):$(ARCH)-latest"
 
 container-name:
@@ -235,14 +271,15 @@ container-name:
 
 manifest: .manifest-$(VERSION) manifest-name
 .manifest-$(VERSION): Dockerfile $(addprefix push-, $(ALL_ARCH))
-	@docker manifest create --amend $(IMAGE):$(VERSION) $(addsuffix -$(VERSION), $(addprefix squat/$(PROJECT):, $(ALL_ARCH)))
+	@docker manifest create --amend $(FULLY_QUALIFIED_IMAGE):$(VERSION) $(addsuffix -$(VERSION), $(addprefix $(FULLY_QUALIFIED_IMAGE):, $(ALL_ARCH)))
 	@$(MAKE) --no-print-directory manifest-annotate-$(VERSION)
-	@docker manifest push $(IMAGE):$(VERSION) > $@
+	@docker manifest push $(FULLY_QUALIFIED_IMAGE):$(VERSION) > $@
 
 manifest-latest: Dockerfile $(addprefix push-latest-, $(ALL_ARCH))
-	@docker manifest create --amend $(IMAGE):latest $(addsuffix -latest, $(addprefix squat/$(PROJECT):, $(ALL_ARCH)))
+	@docker manifest rm $(FULLY_QUALIFIED_IMAGE):latest || echo no old manifest
+	@docker manifest create --amend $(FULLY_QUALIFIED_IMAGE):latest $(addsuffix -latest, $(addprefix $(FULLY_QUALIFIED_IMAGE):, $(ALL_ARCH)))
 	@$(MAKE) --no-print-directory manifest-annotate-latest
-	@docker manifest push $(IMAGE):latest
+	@docker manifest push $(FULLY_QUALIFIED_IMAGE):latest
 	@echo "manifest: $(IMAGE):latest"
 
 manifest-annotate: manifest-annotate-$(VERSION)
@@ -253,7 +290,7 @@ manifest-annotate-%:
 	    annotate=; \
 	    j=0; for da in $(DOCKER_ARCH); do \
 		if [ "$$j" -eq "$$i" ] && [ -n "$$da" ]; then \
-		    annotate="docker manifest annotate $(IMAGE):$* $(IMAGE):$$a-$* --os linux --arch"; \
+		    annotate="docker manifest annotate $(FULLY_QUALIFIED_IMAGE):$* $(FULLY_QUALIFIED_IMAGE):$$a-$* --os linux --arch"; \
 		    k=0; for ea in $$da; do \
 			[ "$$k" = 0 ] && annotate="$$annotate $$ea"; \
 			[ "$$k" != 0 ] && annotate="$$annotate --variant $$ea"; \
@@ -267,20 +304,29 @@ manifest-annotate-%:
 	done
 
 manifest-name:
-	@echo "manifest: $(IMAGE_ROOT):$(VERSION)"
+	@echo "manifest: $(IMAGE):$(VERSION)"
 
 push: .push-$(ARCH)-$(VERSION) push-name
 .push-$(ARCH)-$(VERSION): .container-$(ARCH)-$(VERSION)
-	@docker push $(REGISTRY)/$(IMAGE):$(ARCH)-$(VERSION)
+ifneq ($(REGISTRY),index.docker.io)
+	@docker tag $(IMAGE):$(ARCH)-$(VERSION) $(FULLY_QUALIFIED_IMAGE):$(ARCH)-$(VERSION)
+endif
+	@docker push $(FULLY_QUALIFIED_IMAGE):$(ARCH)-$(VERSION)
 	@docker images -q $(IMAGE):$(ARCH)-$(VERSION) > $@
 
 push-latest: container-latest
-	@docker push $(REGISTRY)/$(IMAGE):$(ARCH)-latest
+	@docker push $(FULLY_QUALIFIED_IMAGE):$(ARCH)-latest
 	@echo "pushed: $(IMAGE):$(ARCH)-latest"
 
 push-name:
 	@echo "pushed: $(IMAGE):$(ARCH)-$(VERSION)"
 
+release: $(RELEASE_BINS)
+$(RELEASE_BINS):
+	@make OS=$(word 2,$(subst -, ,$(@F))) ARCH=$(word 3,$(subst -, ,$(@F)))
+	mkdir -p $(@D)
+	cp bin/$(word 2,$(subst -, ,$(@F)))/$(word 3,$(subst -, ,$(@F)))/kgctl $@
+
 clean: container-clean bin-clean
 	rm -rf .cache
 
@@ -294,6 +340,9 @@ vendor:
 	go mod tidy
 	go mod vendor
 
+$(CONTROLLER_GEN_BINARY):
+	go build -mod=vendor -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen
+
 $(CLIENT_GEN_BINARY):
 	go build -mod=vendor -o $@ k8s.io/code-generator/cmd/client-gen
 
@@ -306,8 +355,11 @@ $(INFORMER_GEN_BINARY):
 $(LISTER_GEN_BINARY):
 	go build -mod=vendor -o $@ k8s.io/code-generator/cmd/lister-gen
 
-$(OPENAPI_GEN_BINARY):
+$(DOCS_GEN_BINARY): cmd/docs-gen/main.go
-	go build -mod=vendor -o $@ k8s.io/kube-openapi/cmd/openapi-gen
+	go build -mod=vendor -o $@ ./cmd/docs-gen
 
-$(GOLINT_BINARY):
+$(STATICCHECK_BINARY):
-	go build -mod=vendor -o $@ golang.org/x/lint/golint
+	go build -mod=vendor -o $@ honnef.co/go/tools/cmd/staticcheck
+
+$(EMBEDMD_BINARY):
+	go build -mod=vendor -o $@ github.com/campoy/embedmd

README.md

@@ -4,17 +4,22 @@
 
 Kilo is a multi-cloud network overlay built on WireGuard and designed for Kubernetes.
 
-[![Build Status](https://travis-ci.org/squat/kilo.svg?branch=master)](https://travis-ci.org/squat/kilo)
+[![Build Status](https://github.com/squat/kilo/workflows/CI/badge.svg)](https://github.com/squat/kilo/actions?query=workflow%3ACI)
 [![Go Report Card](https://goreportcard.com/badge/github.com/squat/kilo)](https://goreportcard.com/report/github.com/squat/kilo)
+[![Docker Pulls](https://img.shields.io/docker/pulls/squat/kilo)](https://hub.docker.com/r/squat/kilo)
+[![Slack](https://img.shields.io/badge/join%20slack-%23kilo-brightgreen.svg)](https://slack.k8s.io/)
 
 ## Overview
 
 Kilo connects nodes in a cluster by providing an encrypted layer 3 network that can span across data centers and public clouds.
+The Pod network created by Kilo is always fully connected, even when the nodes are in different networks or behind NAT.
 By allowing pools of nodes in different locations to communicate securely, Kilo enables the operation of multi-cloud clusters.
 Kilo's design allows clients to VPN to a cluster in order to securely access services running on the cluster.
 In addition to creating multi-cloud clusters, Kilo enables the creation of multi-cluster services, i.e. services that span across different Kubernetes clusters.
 
-## How it works
+An introductory video about Kilo from KubeCon EU 2019 can be found on [youtube](https://www.youtube.com/watch?v=iPz_DAOOCKA).
+
+## How It Works
 
 Kilo uses [WireGuard](https://www.wireguard.com/), a performant and secure VPN, to create a mesh between the different nodes in a cluster.
 The Kilo agent, `kg`, runs on every node in the cluster, setting up the public and private keys for the VPN as well as the necessary rules to route packets between locations.
@@ -26,15 +31,14 @@ This means that if a cluster uses, for example, Flannel for networking, Kilo can
 
 Kilo can be installed on any Kubernetes cluster either pre- or post-bring-up.
 
-### Step 1: install WireGuard
+### Step 1: get WireGuard
 
-Kilo requires the WireGuard kernel module on all nodes in the cluster.
+Kilo requires the WireGuard kernel module to be loaded on all nodes in the cluster.
-For most Linux distributions, this can be installed using the system package manager.
+Starting at Linux 5.6, the kernel includes WireGuard in-tree; Linux distributions with older kernels will need to install WireGuard.
-For Container Linux, WireGuard can be easily installed using a DaemonSet:
+For most Linux distributions, this can be done using the system package manager.
+[See the WireGuard website for up-to-date instructions for installing WireGuard](https://www.wireguard.com/install/).
 
-```shell
+Clusters with nodes on which the WireGuard kernel module cannot be installed can use Kilo by leveraging a [userspace WireGuard implementation](./docs/userspace-wireguard.md).
-kubectl apply -f https://raw.githubusercontent.com/squat/modulus/master/wireguard/daemonset.yaml
-```
 
 ### Step 2: open WireGuard port
 
@@ -68,25 +72,29 @@ Kilo can be installed by deploying a DaemonSet to the cluster.
 To run Kilo on kubeadm:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-kubeadm.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-kubeadm.yaml
 ```
 
 To run Kilo on bootkube:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-bootkube.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-bootkube.yaml
 ```
 
 To run Kilo on Typhoon:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-typhoon.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-typhoon.yaml
 ```
 
 To run Kilo on k3s:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-k3s.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s.yaml
 ```
 
 ## Add-on Mode
@@ -98,15 +106,16 @@ Kilo currently supports running on top of Flannel.
 For example, to run Kilo on a Typhoon cluster running Flannel:
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kilo-typhoon-flannel.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-typhoon-flannel.yaml
 ```
 
-[See the manifests directory for more examples](https://github.com/squat/kilo/tree/master/manifests).
+[See the manifests directory for more examples](https://github.com/squat/kilo/tree/main/manifests).
 
 ## VPN
 
 Kilo also enables peers outside of a Kubernetes cluster to connect to the VPN, allowing cluster applications to securely access external services and permitting developers and support to securely debug cluster resources.
-In order to declare a peer, start by defining a Kilo peer resource:
+In order to declare a peer, start by defining a Kilo Peer resource:
 
 ```shell
 cat <<'EOF' | kubectl apply -f -
@@ -147,7 +156,7 @@ for n in $(kubectl --kubeconfig $KUBECONFIG2 get no -o name | cut -d'/' -f2); do
     kgctl --kubeconfig $KUBECONFIG2 showconf node $n --as-peer -o yaml --allowed-ips $SERVICECIDR2 | kubectl --kubeconfig $KUBECONFIG1 apply -f -
 done
 # Create a Service in cluster2 to mirror the Service in cluster1.
-cat <<'EOF' | kubectl --kubeconfig $KUBECONFIG2 apply -f -
+cat <<EOF | kubectl --kubeconfig $KUBECONFIG2 apply -f -
 apiVersion: v1
 kind: Service
 metadata:

cmd/docs-gen/main.go

@@ -0,0 +1,292 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file was adapted from https://github.com/prometheus-operator/prometheus-operator/blob/master/cmd/po-docgen/api.go.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"os"
+	"reflect"
+	"strings"
+)
+
+const (
+	firstParagraph = `# API
+This document is a reference of the API types introduced by Kilo.
+
+> **Note**: this document is generated from code comments. When contributing a change to this document, please do so by changing the code comments.`
+)
+
+var (
+	links = map[string]string{
+		"metav1.ObjectMeta": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#objectmeta-v1-meta",
+		"metav1.ListMeta":   "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#listmeta-v1-meta",
+	}
+
+	selfLinks = map[string]string{}
+	typesDoc  = map[string]KubeTypes{}
+)
+
+func toSectionLink(name string) string {
+	name = strings.ToLower(name)
+	name = strings.Replace(name, " ", "-", -1)
+	return name
+}
+
+func printTOC(types []KubeTypes) {
+	fmt.Printf("\n## Table of Contents\n")
+	for _, t := range types {
+		strukt := t[0]
+		if len(t) > 1 {
+			fmt.Printf("* [%s](#%s)\n", strukt.Name, toSectionLink(strukt.Name))
+		}
+	}
+}
+
+func printAPIDocs(paths []string) {
+	fmt.Println(firstParagraph)
+
+	types := parseDocumentationFrom(paths)
+	for _, t := range types {
+		strukt := t[0]
+		selfLinks[strukt.Name] = "#" + strings.ToLower(strukt.Name)
+		typesDoc[toLink(strukt.Name)] = t[1:]
+	}
+
+	// we need to parse once more to now add the self links and the inlined fields
+	types = parseDocumentationFrom(paths)
+
+	printTOC(types)
+
+	for _, t := range types {
+		strukt := t[0]
+		if len(t) > 1 {
+			fmt.Printf("\n## %s\n\n%s\n\n", strukt.Name, strukt.Doc)
+
+			fmt.Println("| Field | Description | Scheme | Required |")
+			fmt.Println("| ----- | ----------- | ------ | -------- |")
+			fields := t[1:]
+			for _, f := range fields {
+				fmt.Println("|", f.Name, "|", f.Doc, "|", f.Type, "|", f.Mandatory, "|")
+			}
+			fmt.Println("")
+			fmt.Println("[Back to TOC](#table-of-contents)")
+		}
+	}
+}
+
+// Pair of strings. We need the name of fields and the doc.
+type Pair struct {
+	Name, Doc, Type string
+	Mandatory       bool
+}
+
+// KubeTypes is an array to represent all available types in a parsed file. [0] is for the type itself
+type KubeTypes []Pair
+
+// parseDocumentationFrom gets all types' documentation and returns them as an
+// array. Each type is again represented as an array (we have to use arrays as we
+// need to be sure of the order of the fields). This function returns fields and
+// struct definitions that have no documentation as {name, ""}.
+func parseDocumentationFrom(srcs []string) []KubeTypes {
+	var docForTypes []KubeTypes
+
+	for _, src := range srcs {
+		pkg := astFrom(src)
+
+		for _, kubType := range pkg.Types {
+			if structType, ok := kubType.Decl.Specs[0].(*ast.TypeSpec).Type.(*ast.StructType); ok {
+				var ks KubeTypes
+				ks = append(ks, Pair{kubType.Name, fmtRawDoc(kubType.Doc), "", false})
+
+				for _, field := range structType.Fields.List {
+					// Skip fields that are not tagged.
+					if field.Tag == nil {
+						os.Stderr.WriteString(fmt.Sprintf("Tag is nil, skipping field: %v of type %v\n", field, field.Type))
+						continue
+					}
+					// Treat inlined fields separately as we don't want the original types to appear in the doc.
+					if isInlined(field) {
+						// Skip external types, as we don't want their content to be part of the API documentation.
+						if isInternalType(field.Type) {
+							ks = append(ks, typesDoc[fieldType(field.Type)]...)
+						}
+						continue
+					}
+
+					typeString := fieldType(field.Type)
+					fieldMandatory := fieldRequired(field)
+					if n := fieldName(field); n != "-" {
+						fieldDoc := fmtRawDoc(field.Doc.Text())
+						ks = append(ks, Pair{n, fieldDoc, typeString, fieldMandatory})
+					}
+				}
+				docForTypes = append(docForTypes, ks)
+			}
+		}
+	}
+
+	return docForTypes
+}
+
+func astFrom(filePath string) *doc.Package {
+	fset := token.NewFileSet()
+	m := make(map[string]*ast.File)
+
+	f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments)
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+
+	m[filePath] = f
+	apkg, _ := ast.NewPackage(fset, m, nil, nil)
+
+	return doc.New(apkg, "", 0)
+}
+
+func fmtRawDoc(rawDoc string) string {
+	var buffer bytes.Buffer
+	delPrevChar := func() {
+		if buffer.Len() > 0 {
+			buffer.Truncate(buffer.Len() - 1) // Delete the last " " or "\n"
+		}
+	}
+
+	// Ignore all lines after ---
+	rawDoc = strings.Split(rawDoc, "---")[0]
+
+	for _, line := range strings.Split(rawDoc, "\n") {
+		line = strings.TrimRight(line, " ")
+		leading := strings.TrimLeft(line, " ")
+		switch {
+		case len(line) == 0: // Keep paragraphs
+			delPrevChar()
+			buffer.WriteString("\n\n")
+		case strings.HasPrefix(leading, "TODO"): // Ignore one line TODOs
+		case strings.HasPrefix(leading, "+"): // Ignore instructions to go2idl
+		default:
+			if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
+				delPrevChar()
+				line = "\n" + line + "\n" // Replace it with newline. This is useful when we have a line with: "Example:\n\tJSON-someting..."
+			} else {
+				line += " "
+			}
+			buffer.WriteString(line)
+		}
+	}
+
+	postDoc := strings.TrimRight(buffer.String(), "\n")
+	postDoc = strings.Replace(postDoc, "\\\"", "\"", -1) // replace user's \" to "
+	postDoc = strings.Replace(postDoc, "\"", "\\\"", -1) // Escape "
+	postDoc = strings.Replace(postDoc, "\n", "\\n", -1)
+	postDoc = strings.Replace(postDoc, "\t", "\\t", -1)
+	postDoc = strings.Replace(postDoc, "|", "\\|", -1)
+
+	return postDoc
+}
+
+func toLink(typeName string) string {
+	selfLink, hasSelfLink := selfLinks[typeName]
+	if hasSelfLink {
+		return wrapInLink(typeName, selfLink)
+	}
+
+	link, hasLink := links[typeName]
+	if hasLink {
+		return wrapInLink(typeName, link)
+	}
+
+	return typeName
+}
+
+func wrapInLink(text, link string) string {
+	return fmt.Sprintf("[%s](%s)", text, link)
+}
+
+func isInlined(field *ast.Field) bool {
+	jsonTag := reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+	return strings.Contains(jsonTag, "inline")
+}
+
+func isInternalType(typ ast.Expr) bool {
+	switch typ := typ.(type) {
+	case *ast.SelectorExpr:
+		pkg := typ.X.(*ast.Ident)
+		return strings.HasPrefix(pkg.Name, "monitoring")
+	case *ast.StarExpr:
+		return isInternalType(typ.X)
+	case *ast.ArrayType:
+		return isInternalType(typ.Elt)
+	case *ast.MapType:
+		return isInternalType(typ.Key) && isInternalType(typ.Value)
+	default:
+		return true
+	}
+}
+
+// fieldName returns the name of the field as it should appear in JSON format
+// "-" indicates that this field is not part of the JSON representation
+func fieldName(field *ast.Field) string {
+	jsonTag := reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+	jsonTag = strings.Split(jsonTag, ",")[0]                                              // This can return "-"
+	if jsonTag == "" {
+		if field.Names != nil {
+			return field.Names[0].Name
+		}
+		return field.Type.(*ast.Ident).Name
+	}
+	return jsonTag
+}
+
+// fieldRequired returns whether a field is a required field.
+func fieldRequired(field *ast.Field) bool {
+	jsonTag := ""
+	if field.Tag != nil {
+		jsonTag = reflect.StructTag(field.Tag.Value[1 : len(field.Tag.Value)-1]).Get("json") // Delete first and last quotation
+		return !strings.Contains(jsonTag, "omitempty")
+	}
+
+	return false
+}
+
+func fieldType(typ ast.Expr) string {
+	switch typ := typ.(type) {
+	case *ast.Ident:
+		return toLink(typ.Name)
+	case *ast.StarExpr:
+		return "*" + toLink(fieldType(typ.X))
+	case *ast.SelectorExpr:
+		pkg := typ.X.(*ast.Ident)
+		t := typ.Sel
+		return toLink(pkg.Name + "." + t.Name)
+	case *ast.ArrayType:
+		return "[]" + toLink(fieldType(typ.Elt))
+	case *ast.MapType:
+		return "map[" + toLink(fieldType(typ.Key)) + "]" + toLink(fieldType(typ.Value))
+	default:
+		return ""
+	}
+}
+
+func main() {
+	printAPIDocs(os.Args[1:])
+}

cmd/kg/handlers.go

@@ -0,0 +1,147 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/squat/kilo/pkg/mesh"
+)
+
+type graphHandler struct {
+	mesh        *mesh.Mesh
+	granularity mesh.Granularity
+	hostname    *string
+	subnet      *net.IPNet
+}
+
+func (h *graphHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ns, err := h.mesh.Nodes().List()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to list nodes: %v", err), http.StatusInternalServerError)
+		return
+	}
+	ps, err := h.mesh.Peers().List()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to list peers: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	nodes := make(map[string]*mesh.Node)
+	for _, n := range ns {
+		if n.Ready() {
+			nodes[n.Name] = n
+		}
+	}
+	if len(nodes) == 0 {
+		http.Error(w, "did not find any valid Kilo nodes in the cluster", http.StatusInternalServerError)
+		return
+	}
+	peers := make(map[string]*mesh.Peer)
+	for _, p := range ps {
+		if p.Ready() {
+			peers[p.Name] = p
+		}
+	}
+	topo, err := mesh.NewTopology(nodes, peers, h.granularity, *h.hostname, 0, wgtypes.Key{}, h.subnet, nodes[*h.hostname].PersistentKeepalive, nil)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to create topology: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	dot, err := topo.Dot()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to generate graph: %v", err), http.StatusInternalServerError)
+	}
+
+	buf := bytes.NewBufferString(dot)
+
+	format := r.URL.Query().Get("format")
+	switch format {
+	case "":
+		format = "svg"
+	case "dot", "gv":
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		w.Write(buf.Bytes())
+		return
+
+	case "svg", "png", "bmp", "fig", "gif", "json", "ps":
+		// Accepted format
+
+	default:
+		http.Error(w, "unsupported format", http.StatusInternalServerError)
+		return
+	}
+
+	layout := r.URL.Query().Get("layout")
+	switch layout {
+	case "":
+		layout = "circo"
+
+	case "circo", "dot", "neato", "twopi", "fdp":
+		// Accepted layout
+
+	default:
+		http.Error(w, "unsupported layout", http.StatusInternalServerError)
+		return
+	}
+
+	command := exec.Command("dot", "-K"+layout, "-T"+format)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if _, err = io.Copy(stdin, buf); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err = stdin.Close(); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	output, err := command.Output()
+	if err != nil {
+		http.Error(w, "unable to render graph", http.StatusInternalServerError)
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + format)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	w.Header().Add("content-type", mimeType)
+	w.Write(output)
+}
+
+func healthHandler(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}

cmd/kg/main.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,8 +15,8 @@
 package main
 
 import (
+	"context"
 	"errors"
-	"flag"
 	"fmt"
 	"net"
 	"net/http"
@@ -24,12 +24,15 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
+	"github.com/metalmatze/signal/internalserver"
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/spf13/cobra"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -39,6 +42,7 @@ import (
 	kiloclient "github.com/squat/kilo/pkg/k8s/clientset/versioned"
 	"github.com/squat/kilo/pkg/mesh"
 	"github.com/squat/kilo/pkg/version"
+	"github.com/squat/kilo/pkg/wireguard"
 )
 
 const (
@@ -76,48 +80,79 @@ var (
 	}, ", ")
 )
 
-// Main is the principal function for the binary, wrapped only by `main` for convenience.
+var cmd = &cobra.Command{
-func Main() error {
+	Use:   "kg",
-	backend := flag.String("backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
+	Short: "kg is the Kilo agent",
-	cleanUpIface := flag.Bool("clean-up-interface", false, "Should Kilo delete its interface when it shuts down?")
+	Long: `kg is the Kilo agent.
-	cni := flag.Bool("cni", true, "Should Kilo manage the node's CNI configuration?")
+		It runs on every node of a cluster,
-	cniPath := flag.String("cni-path", mesh.DefaultCNIPath, "Path to CNI config.")
+		setting up the public and private keys for the VPN
-	compatibility := flag.String("compatibility", "", fmt.Sprintf("Should Kilo run in compatibility mode? Possible values: %s", availableCompatibilities))
+		as well as the necessary rules to route packets between locations.`,
-	encapsulate := flag.String("encapsulate", string(encapsulation.Always), fmt.Sprintf("When should Kilo encapsulate packets within a location? Possible values: %s", availableEncapsulations))
+	PreRunE:       preRun,
-	granularity := flag.String("mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
+	RunE:          runRoot,
-	kubeconfig := flag.String("kubeconfig", "", "Path to kubeconfig.")
+	SilenceUsage:  true,
-	hostname := flag.String("hostname", "", "Hostname of the node on which this process is running.")
+	SilenceErrors: true,
-	iface := flag.String("interface", mesh.DefaultKiloInterface, "Name of the Kilo interface to use; if it does not exist, it will be created.")
+}
-	listen := flag.String("listen", ":1107", "The address at which to listen for health and metrics.")
-	local := flag.Bool("local", true, "Should Kilo manage routes within a location?")
-	logLevel := flag.String("log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels))
-	master := flag.String("master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
-	var port uint
-	flag.UintVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
-	subnet := flag.String("subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.")
-	printVersion := flag.Bool("version", false, "Print version and exit")
-	flag.Parse()
 
-	if *printVersion {
+var (
-		fmt.Println(version.Version)
+	backend               string
-		return nil
+	cleanUpIface          bool
-	}
+	createIface           bool
+	cni                   bool
+	cniPath               string
+	compatibility         string
+	encapsulate           string
+	granularity           string
+	hostname              string
+	kubeconfig            string
+	iface                 string
+	listen                string
+	local                 bool
+	master                string
+	mtu                   uint
+	topologyLabel         string
+	port                  int
+	subnet                string
+	resyncPeriod          time.Duration
+	iptablesForwardRule   bool
+	prioritisePrivateAddr bool
 
-	_, s, err := net.ParseCIDR(*subnet)
+	printVersion bool
-	if err != nil {
+	logLevel     string
-		return fmt.Errorf("failed to parse %q as CIDR: %v", *subnet, err)
-	}
 
-	if *hostname == "" {
+	logger   log.Logger
-		var err error
+	registry *prometheus.Registry
-		*hostname, err = os.Hostname()
+)
-		if *hostname == "" || err != nil {
-			return errors.New("failed to determine hostname")
-		}
-	}
 
-	logger := log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
+func init() {
-	switch *logLevel {
+	cmd.Flags().StringVar(&backend, "backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
+	cmd.Flags().BoolVar(&cleanUpIface, "clean-up-interface", false, "Should Kilo delete its interface when it shuts down?")
+	cmd.Flags().BoolVar(&createIface, "create-interface", true, "Should kilo create an interface on startup?")
+	cmd.Flags().BoolVar(&cni, "cni", true, "Should Kilo manage the node's CNI configuration?")
+	cmd.Flags().StringVar(&cniPath, "cni-path", mesh.DefaultCNIPath, "Path to CNI config.")
+	cmd.Flags().StringVar(&compatibility, "compatibility", "", fmt.Sprintf("Should Kilo run in compatibility mode? Possible values: %s", availableCompatibilities))
+	cmd.Flags().StringVar(&encapsulate, "encapsulate", string(encapsulation.Always), fmt.Sprintf("When should Kilo encapsulate packets within a location? Possible values: %s", availableEncapsulations))
+	cmd.Flags().StringVar(&granularity, "mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
+	cmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", "Path to kubeconfig.")
+	cmd.Flags().StringVar(&hostname, "hostname", "", "Hostname of the node on which this process is running.")
+	cmd.Flags().StringVar(&iface, "interface", mesh.DefaultKiloInterface, "Name of the Kilo interface to use; if it does not exist, it will be created.")
+	cmd.Flags().StringVar(&listen, "listen", ":1107", "The address at which to listen for health and metrics.")
+	cmd.Flags().BoolVar(&local, "local", true, "Should Kilo manage routes within a location?")
+	cmd.Flags().StringVar(&master, "master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
+	cmd.Flags().UintVar(&mtu, "mtu", wireguard.DefaultMTU, "The MTU of the WireGuard interface created by Kilo.")
+	cmd.Flags().StringVar(&topologyLabel, "topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
+	cmd.Flags().IntVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
+	cmd.Flags().StringVar(&subnet, "subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.")
+	cmd.Flags().DurationVar(&resyncPeriod, "resync-period", 30*time.Second, "How often should the Kilo controllers reconcile?")
+	cmd.Flags().BoolVar(&iptablesForwardRule, "iptables-forward-rules", false, "Add default accept rules to the FORWARD chain in iptables. Warning: this may break firewalls with a deny all policy and is potentially insecure!")
+	cmd.Flags().BoolVar(&prioritisePrivateAddr, "prioritise-private-addresses", false, "Prefer to assign a private IP address to the node's endpoint.")
+
+	cmd.PersistentFlags().BoolVar(&printVersion, "version", false, "Print version and exit")
+	cmd.PersistentFlags().StringVar(&logLevel, "log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels))
+}
+
+func preRun(_ *cobra.Command, _ []string) error {
+	logger = log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
+	switch logLevel {
 	case logLevelAll:
 		logger = level.NewFilter(logger, level.AllowAll())
 	case logLevelDebug:
@@ -131,78 +166,109 @@ func Main() error {
 	case logLevelNone:
 		logger = level.NewFilter(logger, level.AllowNone())
 	default:
-		return fmt.Errorf("log level %v unknown; possible values are: %s", *logLevel, availableLogLevels)
+		return fmt.Errorf("log level %v unknown; possible values are: %s", logLevel, availableLogLevels)
 	}
 	logger = log.With(logger, "ts", log.DefaultTimestampUTC)
 	logger = log.With(logger, "caller", log.DefaultCaller)
 
-	e := encapsulation.Strategy(*encapsulate)
+	registry = prometheus.NewRegistry()
+	registry.MustRegister(
+		collectors.NewGoCollector(),
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+	)
+
+	return nil
+}
+
+// runRoot is the principal function for the binary.
+func runRoot(_ *cobra.Command, _ []string) error {
+	if printVersion {
+		fmt.Println(version.Version)
+		return nil
+	}
+
+	_, s, err := net.ParseCIDR(subnet)
+	if err != nil {
+		return fmt.Errorf("failed to parse %q as CIDR: %v", subnet, err)
+	}
+
+	if hostname == "" {
+		var err error
+		hostname, err = os.Hostname()
+		if hostname == "" || err != nil {
+			return errors.New("failed to determine hostname")
+		}
+	}
+
+	e := encapsulation.Strategy(encapsulate)
 	switch e {
 	case encapsulation.Never:
 	case encapsulation.CrossSubnet:
 	case encapsulation.Always:
 	default:
-		return fmt.Errorf("encapsulation %v unknown; possible values are: %s", *encapsulate, availableEncapsulations)
+		return fmt.Errorf("encapsulation %v unknown; possible values are: %s", encapsulate, availableEncapsulations)
 	}
 
 	var enc encapsulation.Encapsulator
-	switch *compatibility {
+	switch compatibility {
 	case "flannel":
 		enc = encapsulation.NewFlannel(e)
+	case "cilium":
+		enc = encapsulation.NewCilium(e)
 	default:
 		enc = encapsulation.NewIPIP(e)
 	}
 
-	gr := mesh.Granularity(*granularity)
+	gr := mesh.Granularity(granularity)
 	switch gr {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
 	default:
-		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", *granularity, availableGranularities)
+		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", granularity, availableGranularities)
 	}
 
 	var b mesh.Backend
-	switch *backend {
+	switch backend {
 	case k8s.Backend:
-		config, err := clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
+		config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig)
 		if err != nil {
 			return fmt.Errorf("failed to create Kubernetes config: %v", err)
 		}
 		c := kubernetes.NewForConfigOrDie(config)
 		kc := kiloclient.NewForConfigOrDie(config)
 		ec := apiextensions.NewForConfigOrDie(config)
-		b = k8s.New(c, kc, ec)
+		b = k8s.New(c, kc, ec, topologyLabel, log.With(logger, "component", "k8s backend"))
 	default:
-		return fmt.Errorf("backend %v unknown; possible values are: %s", *backend, availableBackends)
+		return fmt.Errorf("backend %v unknown; possible values are: %s", backend, availableBackends)
 	}
 
-	m, err := mesh.New(b, enc, gr, *hostname, uint32(port), s, *local, *cni, *cniPath, *iface, *cleanUpIface, log.With(logger, "component", "kilo"))
+	if port < 1 || port > 1<<16-1 {
+		return fmt.Errorf("invalid port: port mus be in range [%d:%d], but got %d", 1, 1<<16-1, port)
+	}
+	m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, log.With(logger, "component", "kilo"))
 	if err != nil {
 		return fmt.Errorf("failed to create Kilo mesh: %v", err)
 	}
 
-	r := prometheus.NewRegistry()
+	m.RegisterMetrics(registry)
-	r.MustRegister(
-		prometheus.NewGoCollector(),
-		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
-	)
-	m.RegisterMetrics(r)
 
 	var g run.Group
 	{
+		h := internalserver.NewHandler(
+			internalserver.WithName("Internal Kilo API"),
+			internalserver.WithPrometheusRegistry(registry),
+			internalserver.WithPProf(),
+		)
+		h.AddEndpoint("/health", "Exposes health checks", healthHandler)
+		h.AddEndpoint("/graph", "Exposes Kilo mesh topology graph", (&graphHandler{m, gr, &hostname, s}).ServeHTTP)
 		// Run the HTTP server.
-		mux := http.NewServeMux()
+		l, err := net.Listen("tcp", listen)
-		mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
-			w.WriteHeader(http.StatusOK)
-		})
-		mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{}))
-		l, err := net.Listen("tcp", *listen)
 		if err != nil {
-			return fmt.Errorf("failed to listen on %s: %v", *listen, err)
+			return fmt.Errorf("failed to listen on %s: %v", listen, err)
 		}
 
 		g.Add(func() error {
-			if err := http.Serve(l, mux); err != nil && err != http.ErrServerClosed {
+			if err := http.Serve(l, h); err != nil && err != http.ErrServerClosed {
 				return fmt.Errorf("error: server exited unexpectedly: %v", err)
 			}
 			return nil
@@ -212,15 +278,16 @@ func Main() error {
 	}
 
 	{
+		ctx, cancel := context.WithCancel(context.Background())
 		// Start the mesh.
 		g.Add(func() error {
 			logger.Log("msg", fmt.Sprintf("Starting Kilo network mesh '%v'.", version.Version))
-			if err := m.Run(); err != nil {
+			if err := m.Run(ctx); err != nil {
 				return fmt.Errorf("error: Kilo exited unexpectedly: %v", err)
 			}
 			return nil
 		}, func(error) {
-			m.Stop()
+			cancel()
 		})
 	}
 
@@ -247,8 +314,15 @@ func Main() error {
 	return g.Run()
 }
 
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print the version and exit.",
+	Run:   func(_ *cobra.Command, _ []string) { fmt.Println(version.Version) },
+}
+
 func main() {
-	if err := Main(); err != nil {
+	cmd.AddCommand(webhookCmd, versionCmd)
+	if err := cmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}

cmd/kg/webhook.go

@@ -0,0 +1,273 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/go-kit/kit/log/level"
+	"github.com/oklog/run"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/cobra"
+	v1 "k8s.io/api/admission/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+
+	kilo "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
+	"github.com/squat/kilo/pkg/version"
+)
+
+var webhookCmd = &cobra.Command{
+	Use: "webhook",
+	PreRunE: func(c *cobra.Command, a []string) error {
+		if c.HasParent() {
+			return c.Parent().PreRunE(c, a)
+		}
+		return nil
+	},
+	Short: "webhook starts a HTTPS server to validate updates and creations of Kilo peers.",
+	RunE:  webhook,
+}
+
+var (
+	certPath    string
+	keyPath     string
+	metricsAddr string
+	listenAddr  string
+)
+
+func init() {
+	webhookCmd.Flags().StringVar(&certPath, "cert-file", "", "The path to a certificate file")
+	webhookCmd.Flags().StringVar(&keyPath, "key-file", "", "The path to a key file")
+	webhookCmd.Flags().StringVar(&metricsAddr, "listen-metrics", ":1107", "The metrics server will be listening to that address")
+	webhookCmd.Flags().StringVar(&listenAddr, "listen", ":8443", "The webhook server will be listening to that address")
+}
+
+var deserializer = serializer.NewCodecFactory(runtime.NewScheme()).UniversalDeserializer()
+
+var (
+	validationCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "admission_requests_total",
+			Help: "The number of received admission reviews requests",
+		},
+		[]string{"operation", "response"},
+	)
+	requestCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_requests_total",
+			Help: "The number of received http requests",
+		},
+		[]string{"handler", "method"},
+	)
+	errorCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "errors_total",
+			Help: "The total number of errors",
+		},
+	)
+)
+
+func validationHandler(w http.ResponseWriter, r *http.Request) {
+	level.Debug(logger).Log("msg", "handling request", "source", r.RemoteAddr)
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		errorCounter.Inc()
+		level.Error(logger).Log("err", "failed to parse body from incoming request", "source", r.RemoteAddr)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	var admissionReview v1.AdmissionReview
+
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/json" {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("received Content-Type=%s, expected application/json", contentType)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+
+	response := v1.AdmissionReview{}
+
+	_, gvk, err := deserializer.Decode(body, nil, &admissionReview)
+	if err != nil {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("Request could not be decoded: %v", err)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+	if *gvk != v1.SchemeGroupVersion.WithKind("AdmissionReview") {
+		errorCounter.Inc()
+		msg := "only API v1 is supported"
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+	response.SetGroupVersionKind(*gvk)
+	response.Response = &v1.AdmissionResponse{
+		UID: admissionReview.Request.UID,
+	}
+
+	rawExtension := admissionReview.Request.Object
+	var peer kilo.Peer
+
+	if err := json.Unmarshal(rawExtension.Raw, &peer); err != nil {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("could not unmarshal extension to peer spec: %v:", err)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+
+	if err := peer.Validate(); err == nil {
+		level.Debug(logger).Log("msg", "got valid peer spec", "spec", peer.Spec, "name", peer.ObjectMeta.Name)
+		validationCounter.With(prometheus.Labels{"operation": string(admissionReview.Request.Operation), "response": "allowed"}).Inc()
+		response.Response.Allowed = true
+	} else {
+		level.Debug(logger).Log("msg", "got invalid peer spec", "spec", peer.Spec, "name", peer.ObjectMeta.Name)
+		validationCounter.With(prometheus.Labels{"operation": string(admissionReview.Request.Operation), "response": "denied"}).Inc()
+		response.Response.Result = &metav1.Status{
+			Message: err.Error(),
+		}
+	}
+
+	res, err := json.Marshal(response)
+	if err != nil {
+		errorCounter.Inc()
+		msg := fmt.Sprintf("failed to marshal response: %v", err)
+		level.Error(logger).Log("err", msg)
+		http.Error(w, msg, http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if _, err := w.Write(res); err != nil {
+		level.Error(logger).Log("err", err, "msg", "failed to write response")
+	}
+}
+
+func metricsMiddleWare(path string, next func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		requestCounter.With(prometheus.Labels{"method": r.Method, "handler": path}).Inc()
+		next(w, r)
+	}
+}
+
+func webhook(_ *cobra.Command, _ []string) error {
+	if printVersion {
+		fmt.Println(version.Version)
+		os.Exit(0)
+	}
+	registry.MustRegister(
+		errorCounter,
+		validationCounter,
+		requestCounter,
+	)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer func() {
+		cancel()
+	}()
+	var g run.Group
+	g.Add(run.SignalHandler(ctx, syscall.SIGINT, syscall.SIGTERM))
+	{
+		mm := http.NewServeMux()
+		mm.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+		msrv := &http.Server{
+			Addr:    metricsAddr,
+			Handler: mm,
+		}
+
+		g.Add(
+			func() error {
+				level.Info(logger).Log("msg", "starting metrics server", "address", msrv.Addr)
+				err := msrv.ListenAndServe()
+				level.Info(logger).Log("msg", "metrics server exited", "err", err)
+				return err
+
+			},
+			func(err error) {
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Info(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+				level.Info(logger).Log("msg", "shutting down metrics server gracefully")
+				ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+				defer func() {
+					cancel()
+				}()
+				if err := msrv.Shutdown(ctx); err != nil {
+					level.Error(logger).Log("msg", "failed to shut down metrics server gracefully", "err", err.Error())
+					msrv.Close()
+				}
+			},
+		)
+	}
+
+	{
+		mux := http.NewServeMux()
+		mux.HandleFunc("/validate", metricsMiddleWare("/validate", validationHandler))
+		srv := &http.Server{
+			Addr:    listenAddr,
+			Handler: mux,
+		}
+		g.Add(
+			func() error {
+				level.Info(logger).Log("msg", "starting webhook server", "address", srv.Addr)
+				err := srv.ListenAndServeTLS(certPath, keyPath)
+				level.Info(logger).Log("msg", "webhook server exited", "err", err)
+				return err
+			},
+			func(err error) {
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Info(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+				level.Info(logger).Log("msg", "shutting down webhook server gracefully")
+				ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+				defer func() {
+					cancel()
+				}()
+				if err := srv.Shutdown(ctx); err != nil {
+					level.Error(logger).Log("msg", "failed to shut down webhook server gracefully", "err", err.Error())
+					srv.Close()
+				}
+			},
+		)
+	}
+
+	err := g.Run()
+	var serr run.SignalError
+	if ok := errors.As(err, &serr); ok {
+		return nil
+	}
+	return err
+}

cmd/kgctl/connect_linux.go

@@ -0,0 +1,372 @@
+// Copyright 2022 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/oklog/run"
+	"github.com/spf13/cobra"
+	"golang.zx2c4.com/wireguard/wgctrl"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/squat/kilo/pkg/iproute"
+	"github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
+	"github.com/squat/kilo/pkg/mesh"
+	"github.com/squat/kilo/pkg/route"
+	"github.com/squat/kilo/pkg/wireguard"
+)
+
+var (
+	logLevel    string
+	connectOpts struct {
+		allowedIP           net.IPNet
+		allowedIPs          []net.IPNet
+		privateKey          string
+		cleanUp             bool
+		mtu                 uint
+		resyncPeriod        time.Duration
+		interfaceName       string
+		persistentKeepalive int
+	}
+)
+
+func takeIPNet(_ net.IP, i *net.IPNet, err error) *net.IPNet {
+	if err != nil {
+		panic(err)
+	}
+	return i
+}
+
+func connect() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:          "connect",
+		Args:         cobra.ExactArgs(1),
+		RunE:         runConnect,
+		Short:        "connect to a Kilo cluster as a peer over WireGuard",
+		SilenceUsage: true,
+	}
+	cmd.Flags().IPNetVarP(&connectOpts.allowedIP, "allowed-ip", "a", *takeIPNet(net.ParseCIDR("10.10.10.10/32")), "Allowed IP of the peer.")
+	cmd.Flags().StringSliceVar(&allowedIPs, "allowed-ips", []string{}, "Additional allowed IPs of the cluster, e.g. the service CIDR.")
+	cmd.Flags().StringVar(&logLevel, "log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels))
+	cmd.Flags().StringVar(&connectOpts.privateKey, "private-key", "", "Path to an existing WireGuard private key file.")
+	cmd.Flags().BoolVar(&connectOpts.cleanUp, "clean-up", true, "Should Kilo clean up the routes and interface when it shuts down?")
+	cmd.Flags().UintVar(&connectOpts.mtu, "mtu", uint(1420), "The MTU for the WireGuard interface.")
+	cmd.Flags().DurationVar(&connectOpts.resyncPeriod, "resync-period", 30*time.Second, "How often should Kilo reconcile?")
+	cmd.Flags().StringVarP(&connectOpts.interfaceName, "interface", "i", mesh.DefaultKiloInterface, "Name of the Kilo interface to use; if it does not exist, it will be created.")
+	cmd.Flags().IntVar(&connectOpts.persistentKeepalive, "persistent-keepalive", 10, "How often should WireGuard send keepalives? Setting to 0 will disable sending keepalives.")
+
+	availableLogLevels = strings.Join([]string{
+		logLevelAll,
+		logLevelDebug,
+		logLevelInfo,
+		logLevelWarn,
+		logLevelError,
+		logLevelNone,
+	}, ", ")
+
+	return cmd
+}
+
+func runConnect(cmd *cobra.Command, args []string) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	logger := log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
+	switch logLevel {
+	case logLevelAll:
+		logger = level.NewFilter(logger, level.AllowAll())
+	case logLevelDebug:
+		logger = level.NewFilter(logger, level.AllowDebug())
+	case logLevelInfo:
+		logger = level.NewFilter(logger, level.AllowInfo())
+	case logLevelWarn:
+		logger = level.NewFilter(logger, level.AllowWarn())
+	case logLevelError:
+		logger = level.NewFilter(logger, level.AllowError())
+	case logLevelNone:
+		logger = level.NewFilter(logger, level.AllowNone())
+	default:
+		return fmt.Errorf("log level %s unknown; possible values are: %s", logLevel, availableLogLevels)
+	}
+	logger = log.With(logger, "ts", log.DefaultTimestampUTC)
+	logger = log.With(logger, "caller", log.DefaultCaller)
+	peerName := args[0]
+
+	for i := range allowedIPs {
+		_, aip, err := net.ParseCIDR(allowedIPs[i])
+		if err != nil {
+			return err
+		}
+		connectOpts.allowedIPs = append(connectOpts.allowedIPs, *aip)
+	}
+
+	var privateKey wgtypes.Key
+	var err error
+	if connectOpts.privateKey == "" {
+		privateKey, err = wgtypes.GeneratePrivateKey()
+		if err != nil {
+			return fmt.Errorf("failed to generate private key: %w", err)
+		}
+	} else {
+		raw, err := os.ReadFile(connectOpts.privateKey)
+		if err != nil {
+			return fmt.Errorf("failed to read private key: %w", err)
+		}
+		privateKey, err = wgtypes.ParseKey(string(raw))
+		if err != nil {
+			return fmt.Errorf("failed to parse private key: %w", err)
+		}
+	}
+	publicKey := privateKey.PublicKey()
+	level.Info(logger).Log("msg", "generated public key", "key", publicKey)
+
+	if _, err := opts.kc.KiloV1alpha1().Peers().Get(ctx, peerName, metav1.GetOptions{}); apierrors.IsNotFound(err) {
+		peer := &v1alpha1.Peer{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: peerName,
+			},
+			Spec: v1alpha1.PeerSpec{
+				AllowedIPs:          []string{connectOpts.allowedIP.String()},
+				PersistentKeepalive: connectOpts.persistentKeepalive,
+				PublicKey:           publicKey.String(),
+			},
+		}
+		if _, err := opts.kc.KiloV1alpha1().Peers().Create(ctx, peer, metav1.CreateOptions{}); err != nil {
+			return fmt.Errorf("failed to create peer: %w", err)
+		}
+		level.Info(logger).Log("msg", "created peer", "peer", peerName)
+		if connectOpts.cleanUp {
+			defer func() {
+				ctxWithTimeout, cancelWithTimeout := context.WithTimeout(context.Background(), 10*time.Second)
+				defer cancelWithTimeout()
+				if err := opts.kc.KiloV1alpha1().Peers().Delete(ctxWithTimeout, peerName, metav1.DeleteOptions{}); err != nil {
+					level.Error(logger).Log("err", fmt.Sprintf("failed to delete peer: %v", err))
+				} else {
+					level.Info(logger).Log("msg", "deleted peer", "peer", peerName)
+				}
+			}()
+		}
+
+	} else if err != nil {
+		return fmt.Errorf("failed to get peer: %w", err)
+	}
+
+	iface, _, err := wireguard.New(connectOpts.interfaceName, connectOpts.mtu)
+	if err != nil {
+		return fmt.Errorf("failed to create wg interface: %w", err)
+	}
+	level.Info(logger).Log("msg", "created WireGuard interface", "name", connectOpts.interfaceName, "index", iface)
+
+	table := route.NewTable()
+	if connectOpts.cleanUp {
+		defer cleanUp(iface, table, logger)
+	}
+
+	if err := iproute.SetAddress(iface, &connectOpts.allowedIP); err != nil {
+		return err
+	}
+	level.Info(logger).Log("msg", "set IP address of WireGuard interface", "IP", connectOpts.allowedIP.String())
+
+	if err := iproute.Set(iface, true); err != nil {
+		return err
+	}
+
+	var g run.Group
+	g.Add(run.SignalHandler(ctx, syscall.SIGINT, syscall.SIGTERM))
+
+	{
+		g.Add(
+			func() error {
+				errCh, err := table.Run(ctx.Done())
+				if err != nil {
+					return fmt.Errorf("failed to watch for route table updates: %w", err)
+				}
+				for {
+					select {
+					case err, ok := <-errCh:
+						if ok {
+							level.Error(logger).Log("err", err.Error())
+						} else {
+							return nil
+						}
+					case <-ctx.Done():
+						return nil
+					}
+				}
+			},
+			func(err error) {
+				cancel()
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Debug(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+			},
+		)
+	}
+	{
+		g.Add(
+			func() error {
+				level.Info(logger).Log("msg", "starting syncer")
+				for {
+					if err := sync(table, peerName, privateKey, iface, logger); err != nil {
+						level.Error(logger).Log("msg", "failed to sync", "err", err.Error())
+					}
+					select {
+					case <-time.After(connectOpts.resyncPeriod):
+					case <-ctx.Done():
+						return nil
+					}
+				}
+			}, func(err error) {
+				cancel()
+				var serr run.SignalError
+				if ok := errors.As(err, &serr); ok {
+					level.Debug(logger).Log("msg", "received signal", "signal", serr.Signal.String(), "err", err.Error())
+				} else {
+					level.Error(logger).Log("msg", "received error", "err", err.Error())
+				}
+			})
+	}
+
+	err = g.Run()
+	var serr run.SignalError
+	if ok := errors.As(err, &serr); ok {
+		return nil
+	}
+	return err
+}
+
+func cleanUp(iface int, t *route.Table, logger log.Logger) {
+	if err := iproute.Set(iface, false); err != nil {
+		level.Error(logger).Log("err", fmt.Sprintf("failed to set WireGuard interface down: %v", err))
+	}
+	if err := iproute.RemoveInterface(iface); err != nil {
+		level.Error(logger).Log("err", fmt.Sprintf("failed to remove WireGuard interface: %v", err))
+	}
+	if err := t.CleanUp(); err != nil {
+		level.Error(logger).Log("failed to clean up routes: %v", err)
+	}
+}
+
+func sync(table *route.Table, peerName string, privateKey wgtypes.Key, iface int, logger log.Logger) error {
+	ns, err := opts.backend.Nodes().List()
+	if err != nil {
+		return fmt.Errorf("failed to list nodes: %w", err)
+	}
+	for _, n := range ns {
+		_, err := n.Endpoint.UDPAddr(true)
+		if err != nil {
+			return err
+		}
+	}
+	ps, err := opts.backend.Peers().List()
+	if err != nil {
+		return fmt.Errorf("failed to list peers: %w", err)
+	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
+	}
+	var hostname string
+	var subnet *net.IPNet
+	nodes := make(map[string]*mesh.Node)
+	var nodeNames []string
+	for _, n := range ns {
+		if n.Ready() {
+			nodes[n.Name] = n
+			hostname = n.Name
+			nodeNames = append(nodeNames, n.Name)
+		}
+		if n.WireGuardIP != nil && subnet == nil {
+			subnet = n.WireGuardIP
+		}
+	}
+	if len(nodes) == 0 {
+		return errors.New("did not find any valid Kilo nodes in the cluster")
+	}
+	if subnet == nil {
+		return errors.New("did not find a valid Kilo subnet on any node")
+	}
+	subnet.IP = subnet.IP.Mask(subnet.Mask)
+	sort.Strings(nodeNames)
+	nodes[nodeNames[0]].AllowedLocationIPs = append(nodes[nodeNames[0]].AllowedLocationIPs, connectOpts.allowedIPs...)
+	peers := make(map[string]*mesh.Peer)
+	for _, p := range ps {
+		if p.Ready() {
+			peers[p.Name] = p
+		}
+	}
+	if _, ok := peers[peerName]; !ok {
+		return fmt.Errorf("did not find any peer named %q in the cluster", peerName)
+	}
+
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, wgtypes.Key{}, subnet, *peers[peerName].PersistentKeepaliveInterval, logger)
+	if err != nil {
+		return fmt.Errorf("failed to create topology: %w", err)
+	}
+	conf := t.PeerConf(peerName)
+	conf.PrivateKey = &privateKey
+	conf.ListenPort = &opts.port
+
+	wgClient, err := wgctrl.New()
+	if err != nil {
+		return err
+	}
+	defer wgClient.Close()
+
+	current, err := wgClient.Device(connectOpts.interfaceName)
+	if err != nil {
+		return err
+	}
+
+	var equal bool
+	var diff string
+	equal, diff = conf.Equal(current)
+	if !equal {
+		// If the key is empty, then it's the first time we are running
+		// so don't bother printing a diff.
+		if current.PrivateKey != [wgtypes.KeyLen]byte{} {
+			level.Info(logger).Log("msg", "WireGuard configurations are different", "diff", diff)
+		}
+		level.Debug(logger).Log("msg", "setting WireGuard config", "config", conf.WGConfig())
+		if err := wgClient.ConfigureDevice(connectOpts.interfaceName, conf.WGConfig()); err != nil {
+			return err
+		}
+	}
+
+	if err := table.Set(t.PeerRoutes(peerName, iface, connectOpts.allowedIPs)); err != nil {
+		return fmt.Errorf("failed to update route table: %w", err)
+	}
+
+	return nil
+}

cmd/kgctl/connect_other.go

@@ -1,4 +1,4 @@
-// Copyright 2015-2017 CNI authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,23 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !linux
 // +build !linux
 
-package ip
+package main
 
 import (
-	"net"
+	"errors"
 
-	"github.com/containernetworking/cni/pkg/types"
+	"github.com/spf13/cobra"
-	"github.com/vishvananda/netlink"
 )
 
-// AddRoute adds a universally-scoped route to a device.
+func connect() *cobra.Command {
-func AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error {
+	cmd := &cobra.Command{
-	return types.NotImplementedError
+		Use:   "connect",
-}
+		Short: "not supporred on this OS",
-
+		RunE: func(_ *cobra.Command, _ []string) error {
-// AddHostRoute adds a host-scoped route to a device.
+			return errors.New("this command is not supported on this OS")
-func AddHostRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error {
+		},
-	return types.NotImplementedError
+	}
+	return cmd
 }

cmd/kgctl/graph.go

@@ -18,6 +18,8 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
 	"github.com/squat/kilo/pkg/mesh"
 )
 
@@ -32,12 +34,17 @@ func graph() *cobra.Command {
 func runGraph(_ *cobra.Command, _ []string) error {
 	ns, err := opts.backend.Nodes().List()
 	if err != nil {
-		return fmt.Errorf("failed to list nodes: %v", err)
+		return fmt.Errorf("failed to list nodes: %w", err)
 	}
 	ps, err := opts.backend.Peers().List()
 	if err != nil {
-		return fmt.Errorf("failed to list peers: %v", err)
+		return fmt.Errorf("failed to list peers: %w", err)
 	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
+	}
+
 	var hostname string
 	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
@@ -60,13 +67,13 @@ func runGraph(_ *cobra.Command, _ []string) error {
 			peers[p.Name] = p
 		}
 	}
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, wgtypes.Key{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create topology: %v", err)
+		return fmt.Errorf("failed to create topology: %w", err)
 	}
 	g, err := t.Dot()
 	if err != nil {
-		return fmt.Errorf("failed to generate graph: %v", err)
+		return fmt.Errorf("failed to generate graph: %w", err)
 	}
 	fmt.Println(g)
 	return nil

cmd/kgctl/main.go

@@ -15,10 +15,13 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
+	"github.com/go-kit/kit/log"
 	"github.com/spf13/cobra"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
@@ -46,6 +49,7 @@ var (
 	availableGranularities = strings.Join([]string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.AutoGranularity),
 	}, ", ")
 	availableLogLevels = strings.Join([]string{
 		logLevelAll,
@@ -58,42 +62,49 @@ var (
 	opts struct {
 		backend     mesh.Backend
 		granularity mesh.Granularity
-		port        uint32
+		kc          kiloclient.Interface
+		port        int
 	}
 	backend       string
 	granularity   string
 	kubeconfig    string
+	topologyLabel string
 )
 
-func runRoot(_ *cobra.Command, _ []string) error {
+func runRoot(c *cobra.Command, _ []string) error {
+	if opts.port < 1 || opts.port > 1<<16-1 {
+		return fmt.Errorf("invalid port: port mus be in range [%d:%d], but got %d", 1, 1<<16-1, opts.port)
+	}
+
 	opts.granularity = mesh.Granularity(granularity)
 	switch opts.granularity {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.AutoGranularity:
 	default:
-		return fmt.Errorf("mesh granularity %v unknown; posible values are: %s", granularity, availableGranularities)
+		return fmt.Errorf("mesh granularity %s unknown; posible values are: %s", granularity, availableGranularities)
 	}
 
 	switch backend {
 	case k8s.Backend:
 		config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 		if err != nil {
-			return fmt.Errorf("failed to create Kubernetes config: %v", err)
+			return fmt.Errorf("failed to create Kubernetes config: %w", err)
 		}
 		c := kubernetes.NewForConfigOrDie(config)
-		kc := kiloclient.NewForConfigOrDie(config)
+		opts.kc = kiloclient.NewForConfigOrDie(config)
 		ec := apiextensions.NewForConfigOrDie(config)
-		opts.backend = k8s.New(c, kc, ec)
+		opts.backend = k8s.New(c, opts.kc, ec, topologyLabel, log.NewNopLogger())
 	default:
-		return fmt.Errorf("backend %v unknown; posible values are: %s", backend, availableBackends)
+		return fmt.Errorf("backend %s unknown; posible values are: %s", backend, availableBackends)
 	}
 
-	if err := opts.backend.Nodes().Init(make(chan struct{})); err != nil {
+	if err := opts.backend.Nodes().Init(c.Context()); err != nil {
-		return fmt.Errorf("failed to initialize node backend: %v", err)
+		return fmt.Errorf("failed to initialize node backend: %w", err)
 	}
 
-	if err := opts.backend.Peers().Init(make(chan struct{})); err != nil {
+	if err := opts.backend.Peers().Init(c.Context()); err != nil {
-		return fmt.Errorf("failed to initialize peer backend: %v", err)
+		return fmt.Errorf("failed to initialize peer backend: %w", err)
 	}
 	return nil
 }
@@ -105,15 +116,22 @@ func main() {
 		Long:              "",
 		PersistentPreRunE: runRoot,
 		Version:           version.Version,
+		SilenceErrors:     true,
 	}
 	cmd.PersistentFlags().StringVar(&backend, "backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
-	cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
+	cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.AutoGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
-	cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig.")
+	defaultKubeconfig := os.Getenv("KUBECONFIG")
-	cmd.PersistentFlags().Uint32Var(&opts.port, "port", mesh.DefaultKiloPort, "The WireGuard port over which the nodes communicate.")
+	if _, err := os.Stat(defaultKubeconfig); os.IsNotExist(err) {
+		defaultKubeconfig = filepath.Join(os.Getenv("HOME"), ".kube/config")
+	}
+	cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", defaultKubeconfig, "Path to kubeconfig.")
+	cmd.PersistentFlags().IntVar(&opts.port, "port", mesh.DefaultKiloPort, "The WireGuard port over which the nodes communicate.")
+	cmd.PersistentFlags().StringVar(&topologyLabel, "topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
 
 	for _, subCmd := range []*cobra.Command{
 		graph(),
 		showConf(),
+		connect(),
 	} {
 		cmd.AddCommand(subCmd)
 	}
@@ -123,3 +141,20 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func determineGranularity(gr mesh.Granularity, ns []*mesh.Node) (mesh.Granularity, error) {
+	if gr == mesh.AutoGranularity {
+		if len(ns) == 0 {
+			return gr, errors.New("could not get any nodes")
+		}
+		ret := mesh.Granularity(ns[0].Granularity)
+		switch ret {
+		case mesh.LogicalGranularity:
+		case mesh.FullGranularity:
+		default:
+			return ret, fmt.Errorf("mesh granularity %s is not supported", opts.granularity)
+		}
+		return ret, nil
+	}
+	return gr, nil
+}

cmd/kgctl/showconf.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,14 +15,15 @@
 package main
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"net"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -47,7 +48,7 @@ var (
 	}, ", ")
 	allowedIPs   []string
 	showConfOpts struct {
-		allowedIPs []*net.IPNet
+		allowedIPs []net.IPNet
 		serializer *json.Serializer
 		output     string
 		asPeer     bool
@@ -82,14 +83,14 @@ func runShowConf(c *cobra.Command, args []string) error {
 	case outputFormatYAML:
 		showConfOpts.serializer = json.NewYAMLSerializer(json.DefaultMetaFactory, peerCreatorTyper{}, peerCreatorTyper{})
 	default:
-		return fmt.Errorf("output format %v unknown; posible values are: %s", showConfOpts.output, availableOutputFormats)
+		return fmt.Errorf("output format %s unknown; posible values are: %s", showConfOpts.output, availableOutputFormats)
 	}
 	for i := range allowedIPs {
 		_, aip, err := net.ParseCIDR(allowedIPs[i])
 		if err != nil {
 			return fmt.Errorf("allowed-ips must contain only valid CIDRs; got %q", allowedIPs[i])
 		}
-		showConfOpts.allowedIPs = append(showConfOpts.allowedIPs, aip)
+		showConfOpts.allowedIPs = append(showConfOpts.allowedIPs, *aip)
 	}
 	return runRoot(c, args)
 }
@@ -115,11 +116,15 @@ func showConfPeer() *cobra.Command {
 func runShowConfNode(_ *cobra.Command, args []string) error {
 	ns, err := opts.backend.Nodes().List()
 	if err != nil {
-		return fmt.Errorf("failed to list nodes: %v", err)
+		return fmt.Errorf("failed to list nodes: %w", err)
 	}
 	ps, err := opts.backend.Peers().List()
 	if err != nil {
-		return fmt.Errorf("failed to list peers: %v", err)
+		return fmt.Errorf("failed to list peers: %w", err)
+	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
 	}
 	hostname := args[0]
 	subnet := mesh.DefaultKiloSubnet
@@ -147,14 +152,14 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		}
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, int(opts.port), wgtypes.Key{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create topology: %v", err)
+		return fmt.Errorf("failed to create topology: %w", err)
 	}
 
 	var found bool
 	for _, p := range t.PeerConf("").Peers {
-		if bytes.Equal(p.PublicKey, nodes[hostname].Key) {
+		if p.PublicKey == nodes[hostname].Key {
 			found = true
 			break
 		}
@@ -167,7 +172,7 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 	if !showConfOpts.asPeer {
 		c, err := t.Conf().Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -178,6 +183,9 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		fallthrough
 	case outputFormatYAML:
 		p := t.AsPeer()
+		if p == nil {
+			return errors.New("cannot generate config from nil peer")
+		}
 		p.AllowedIPs = append(p.AllowedIPs, showConfOpts.allowedIPs...)
 		p.DeduplicateIPs()
 		k8sp := translatePeer(p)
@@ -185,13 +193,16 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		return showConfOpts.serializer.Encode(k8sp, os.Stdout)
 	case outputFormatWireGuard:
 		p := t.AsPeer()
+		if p == nil {
+			return errors.New("cannot generate config from nil peer")
+		}
 		p.AllowedIPs = append(p.AllowedIPs, showConfOpts.allowedIPs...)
 		p.DeduplicateIPs()
 		c, err := (&wireguard.Conf{
-			Peers: []*wireguard.Peer{p},
+			Peers: []wireguard.Peer{*p},
 		}).Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -202,11 +213,15 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 func runShowConfPeer(_ *cobra.Command, args []string) error {
 	ns, err := opts.backend.Nodes().List()
 	if err != nil {
-		return fmt.Errorf("failed to list nodes: %v", err)
+		return fmt.Errorf("failed to list nodes: %w", err)
 	}
 	ps, err := opts.backend.Peers().List()
 	if err != nil {
-		return fmt.Errorf("failed to list peers: %v", err)
+		return fmt.Errorf("failed to list peers: %w", err)
+	}
+	// Obtain the Granularity by looking at the annotation of the first node.
+	if opts.granularity, err = determineGranularity(opts.granularity, ns); err != nil {
+		return fmt.Errorf("failed to determine granularity: %w", err)
 	}
 	var hostname string
 	subnet := mesh.DefaultKiloSubnet
@@ -236,14 +251,18 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		return fmt.Errorf("did not find any peer named %q in the cluster", peer)
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet, peers[peer].PersistentKeepalive)
+	pka := time.Duration(0)
+	if p := peers[peer].PersistentKeepaliveInterval; p != nil {
+		pka = *p
+	}
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, wgtypes.Key{}, subnet, pka, nil)
 	if err != nil {
-		return fmt.Errorf("failed to create topology: %v", err)
+		return fmt.Errorf("failed to create topology: %w", err)
 	}
 	if !showConfOpts.asPeer {
 		c, err := t.PeerConf(peer).Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -264,10 +283,10 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		p.AllowedIPs = append(p.AllowedIPs, showConfOpts.allowedIPs...)
 		p.DeduplicateIPs()
 		c, err := (&wireguard.Conf{
-			Peers: []*wireguard.Peer{p},
+			Peers: []wireguard.Peer{*p},
 		}).Bytes()
 		if err != nil {
-			return fmt.Errorf("failed to generate configuration: %v", err)
+			return fmt.Errorf("failed to generate configuration: %w", err)
 		}
 		_, err = os.Stdout.Write(c)
 		return err
@@ -276,6 +295,7 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 }
 
 // translatePeer translates a wireguard.Peer to a Peer CRD.
+// TODO this function has many similarities to peerBackend.Set(name, peer)
 func translatePeer(peer *wireguard.Peer) *v1alpha1.Peer {
 	if peer == nil {
 		return &v1alpha1.Peer{}
@@ -283,29 +303,33 @@ func translatePeer(peer *wireguard.Peer) *v1alpha1.Peer {
 	var aips []string
 	for _, aip := range peer.AllowedIPs {
 		// Skip any invalid IPs.
-		if aip == nil {
+		// TODO all IPs should be valid, so no need to skip here?
+		if aip.String() == (&net.IPNet{}).String() {
 			continue
 		}
 		aips = append(aips, aip.String())
 	}
 	var endpoint *v1alpha1.PeerEndpoint
-	if peer.Endpoint != nil && peer.Endpoint.Port > 0 && peer.Endpoint.IP != nil {
+	if peer.Endpoint.Port() > 0 || !peer.Endpoint.HasDNS() {
 		endpoint = &v1alpha1.PeerEndpoint{
-			IP:   peer.Endpoint.IP.String(),
+			DNSOrIP: v1alpha1.DNSOrIP{
-			Port: peer.Endpoint.Port,
+				IP:  peer.Endpoint.IP().String(),
+				DNS: peer.Endpoint.DNS(),
+			},
+			Port: uint32(peer.Endpoint.Port()),
 		}
 	}
 	var key string
-	if len(peer.PublicKey) > 0 {
+	if peer.PublicKey != (wgtypes.Key{}) {
-		key = string(peer.PublicKey)
+		key = peer.PublicKey.String()
 	}
 	var psk string
-	if len(peer.PresharedKey) > 0 {
+	if peer.PresharedKey != nil {
-		psk = string(peer.PresharedKey)
+		psk = peer.PresharedKey.String()
 	}
 	var pka int
-	if peer.PersistentKeepalive > 0 {
+	if peer.PersistentKeepaliveInterval != nil && *peer.PersistentKeepaliveInterval > time.Duration(0) {
-		pka = peer.PersistentKeepalive
+		pka = int(*peer.PersistentKeepaliveInterval)
 	}
 	return &v1alpha1.Peer{
 		TypeMeta: metav1.TypeMeta{

docs/annotations.md

@@ -5,10 +5,11 @@ The following annotations can be added to any Kubernetes Node object to configur
 |Name|type|examples|
 |----|----|-------|
 |[kilo.squat.ai/force-endpoint](#force-endpoint)|host:port|`55.55.55.55:51820`, `example.com:1337`|
-|[kilo.squat.ai/force-internal-ip](#force-internal-ip)|CIDR|`55.55.55.55/32`|
+|[kilo.squat.ai/force-internal-ip](#force-internal-ip)|CIDR|`55.55.55.55/32`, `"-"`,`""`|
 |[kilo.squat.ai/leader](#leader)|string|`""`, `true`|
 |[kilo.squat.ai/location](#location)|string|`gcp-east`, `lab`|
 |[kilo.squat.ai/persistent-keepalive](#persistent-keepalive)|uint|`10`|
+|[kilo.squat.ai/allowed-location-ips](#allowed-location-ips)|CIDR|`66.66.66.66/32`|
 
 ### force-endpoint
 In order to create links between locations, Kilo requires at least one node in each location to have an endpoint, ie a `host:port` combination, that is routable from the other locations.
@@ -25,6 +26,7 @@ Kilo routes packets destined for nodes inside the same logical location using th
 The Kilo agent running on each node will use heuristics to automatically detect a private IP address for the node; however, in some circumstances it may be necessary to explicitly configure the IP address, for example:
  * _multiple private IP addresses_: if a node has multiple private IPs but one is preferred, then the preferred IP address should be specified;
  * _IPv6_: if a node has both private IPv4 and IPv6 addresses and the Kilo network should operate over IPv6, then the IPv6 address should be specified.
+ * _disable private IP with "-" or ""_: a node has a private and public address, but the private address ought to be ignored.
 
 ### leader
 By default, Kilo creates a network mesh at the data-center granularity.
@@ -34,7 +36,7 @@ In some situations it may be desirable to manually select the leader for a locat
  * _firewall_: Kilo requires an open UDP port, which defaults to 51820, to communicate between locations; if only one node is configured to have that port open, then that node should be given the leader annotation;
  * _bandwidth_: if certain nodes in the cluster have a higher bandwidth or lower latency Internet connection, then those nodes should be given the leader annotation.
 
-_Note_: multiple nodes within a single location can be given the leader annotation; in this case, Kilo will select one leader from the set of annotated nodes. 
+> **Note**: multiple nodes within a single location can be given the leader annotation; in this case, Kilo will select one leader from the set of annotated nodes.
 
 ### location
 Kilo allows nodes in different logical or physical locations to route packets to one-another.
@@ -42,7 +44,7 @@ In order to know what connections to create, Kilo needs to know which nodes are
 Kilo will try to infer each node's location from the [topology.kubernetes.io/region](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#topologykubernetesioregion) node label.
 If the label is not present for a node, for example if running a bare-metal cluster or on an unsupported cloud provider, then the location annotation should be specified.
 
-_Note_: all nodes without a defined location will be considered to be in the default location `""`.
+> **Note**: all nodes without a defined location will be considered to be in the default location `""`.
 
 ### persistent-keepalive
 In certain deployments, cluster nodes may be located behind NAT or a firewall, e.g. edge nodes located behind a commodity router.
@@ -51,3 +53,10 @@ In order for a node behind NAT to receive packets from nodes outside of the NATe
 The frequency of emission of these keepalive packets can be controlled by setting the persistent-keepalive annotation on the node behind NAT.
 The annotated node will use the specified value will as the persistent-keepalive interval for all of its peers.
 For more background, [see the WireGuard documentation on NAT and firewall traversal](https://www.wireguard.com/quickstart/#nat-and-firewall-traversal-persistence).
+
+### allowed-location-ips
+It is possible to add allowed-location-ips to a location by annotating any node within that location.
+Adding allowed-location-ips to a location makes these IPs routable from other locations as well.
+
+In an example deployment of Kilo with two locations A and B, a printer in location A can be accessible from nodes and pods in location B.
+Additionally, Kilo Peers can use the printer in location A.

docs/api.md

@@ -0,0 +1,69 @@
+# API
+This document is a reference of the API types introduced by Kilo.
+
+> **Note**: this document is generated from code comments. When contributing a change to this document, please do so by changing the code comments.
+
+## Table of Contents
+* [DNSOrIP](#dnsorip)
+* [Peer](#peer)
+* [PeerEndpoint](#peerendpoint)
+* [PeerList](#peerlist)
+* [PeerSpec](#peerspec)
+
+## DNSOrIP
+
+DNSOrIP represents either a DNS name or an IP address. When both are given, the IP address, as it is more specific, override the DNS name.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| dns | DNS must be a valid RFC 1123 subdomain. | string | false |
+| ip | IP must be a valid IP address. | string | false |
+
+[Back to TOC](#table-of-contents)
+
+## Peer
+
+Peer is a WireGuard peer that should have access to the VPN.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| metadata | Standard object’s metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#metadata | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#objectmeta-v1-meta) | false |
+| spec | Specification of the desired behavior of the Kilo Peer. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status | [PeerSpec](#peerspec) | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerEndpoint
+
+PeerEndpoint represents a WireGuard endpoint, which is an IP:port tuple.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| dnsOrIP | DNSOrIP is a DNS name or an IP address. | [DNSOrIP](#dnsorip) | true |
+| port | Port must be a valid port number. | uint32 | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerList
+
+PeerList is a list of peers.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| metadata | Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | [metav1.ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#listmeta-v1-meta) | false |
+| items | List of peers. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md | [][Peer](#peer) | true |
+
+[Back to TOC](#table-of-contents)
+
+## PeerSpec
+
+PeerSpec is the description and configuration of a peer.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| allowedIPs | AllowedIPs is the list of IP addresses that are allowed for the given peer's tunnel. | []string | true |
+| endpoint | Endpoint is the initial endpoint for connections to the peer. | *[PeerEndpoint](#peerendpoint) | false |
+| persistentKeepalive | PersistentKeepalive is the interval in seconds of the emission of keepalive packets by the peer. This defaults to 0, which disables the feature. | int | false |
+| presharedKey | PresharedKey is the optional symmetric encryption key for the peer. | string | false |
+| publicKey | PublicKey is the WireGuard public key for the peer. | string | true |
+
+[Back to TOC](#table-of-contents)

docs/building_kilo.md

@@ -0,0 +1,100 @@
+# Build and Test Kilo
+
+This document describes how you can build and test Kilo.
+
+To follow along, you need to install the following utilities:
+ - `go` not for building but formatting the code and running unit tests
+ - `make`
+ - `jq`
+ - `git`
+ - `curl`
+ - `docker`
+
+## Getting Started
+
+Clone the Repository and `cd` into it.
+```shell
+git clone https://github.com/squat/kilo.git
+cd kilo
+```
+
+## Build
+
+For consistency, the Kilo binaries are compiled in a Docker container, so make sure the `docker` package is installed and the daemon is running.
+
+### Compile Binaries
+
+To compile the `kg` and `kgctl` binaries run:
+```shell
+make
+```
+Binaries are always placed in a directory corresponding to the local system's OS and architecture following the pattern `bin/<os>/<architecture>/`, so on an AMD64 machine running Linux, the binaries will be stored in `bin/linux/amd64/`.
+
+You can build the binaries for a different architecture by setting the `ARCH` environment variable before invoking `make`, e.g.:
+```shell
+ARCH=<arm|arm64|amd64> make
+```
+
+Likewise, to build `kg` for another OS, set the `OS` environment variable before invoking `make`:
+```shell
+OS=<windows|darwin|linux> make
+```
+## Test
+
+To execute the unit tests, run:
+```shell
+make unit
+```
+
+To lint the code in the repository, run:
+```shell
+make lint
+```
+
+To execute basic end to end tests, run:
+```shell
+make e2e
+```
+> **Note**: The end to end tests are currently flaky, so try running them again if they fail.
+
+To instead run all of the tests with a single command, run:
+```shell
+make test
+```
+
+## Build and Push the Container Images
+
+If you want to build containers for a processor architecture that is different from your computer's, then you will first need to configure QEMU as the interpreter for binaries built for non-native architectures:
+```shell
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+
+Set the `$IMAGE` environment variable to `<your Docker Hub user name>/kilo`.
+This way the generated container images and manifests will be named accordingly.
+By skipping this step, you will be able to tag images but will not be able to push the containers and manifests to your own Docker Hub.
+```shell
+export IMAGE=<docker hub user name>/kilo
+```
+
+If you want to use a different container registry, run:
+```shell
+export REGISTRY=<your registry without a trailing slash>
+```
+
+To build containers with the `kg` image for `arm`, `arm64` and `amd64`, run:
+```shell
+make all-container
+```
+
+Push the container images and build a manifest with:
+```shell
+make manifest
+```
+
+To tag and push the manifest with `latest`, run:
+```shell
+make manifest-latest
+```
+
+Now you can deploy the custom build of Kilo to your cluster.
+If you are already running Kilo, change the image from `squat/kilo` to `[registry/]<username>/kilo[:sha]`.

docs/building_website.md

@@ -0,0 +1,44 @@
+# Build and Test the Website
+
+You may have noticed that the `markdown` files in the `/docs` directory are also displayed on [Kilo's website](https://kilo.squat.ai/).
+If you want to add documentation to Kilo, you can start a local webserver to check out how the website would look like.
+
+## Requirements
+
+Install [yarn](https://yarnpkg.com/getting-started/install).
+
+## Build and Run
+
+The markdown files for the website are located in `/website/docs` and are generated from the like-named markdown files in the `/docs` directory and from the corresponding header files without the `.md` extension in the `/website/docs` directory.
+To generate the markdown files in `/website/docs`, run:
+```shell
+make website/docs/README.md
+```
+
+Next, build the website itself by installing the `node_modules` and building the website's HTML from the generated markdown:
+```shell
+make website/build/index.html
+```
+
+Now, start the website server with:
+```shell
+yarn --cwd website start
+```
+This command should have opened a browser window with the website; if not, open your browser and point it to `http://localhost:3000`.
+
+If you make changes to any of the markdown files in `/docs` and want to reload the local `node` server, run:
+```shell
+make website/docs/README.md -B
+```
+
+You can execute the above while the node server is running and the website will be rebuilt and reloaded automatically.
+
+## Add a New File to the Docs
+
+If you add a new file to the `/docs` directory, you also need to create a corresponding header file containing the front-matter in `/website/docs/`.
+Then, regenerate the markdown for the website with the command:
+```shell
+make website/docs/README.md
+```
+Edit `/website/sidebars.js` accordingly.
+> **Note**: The `id` in the header file `/website/docs/<new file>` must match the `id` specified in `website/sidebars.js`.

docs/grafana/kilo.json

@@ -0,0 +1,962 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.5.4"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 12,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (pod) (rate(wireguard_received_bytes_total[1h])) + sum by (pod) (rate(wireguard_sent_bytes_total[1h]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [
+        {
+          "$$hashKey": "object:64",
+          "colorMode": "background6",
+          "fill": true,
+          "fillColor": "rgba(234, 112, 112, 0.12)",
+          "line": false,
+          "lineColor": "rgba(237, 46, 24, 0.60)",
+          "op": "time"
+        }
+      ],
+      "timeShift": null,
+      "title": "Throughput",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:42",
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:43",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "(sum(rate(wireguard_sent_bytes_total[5m])) - sum(rate(wireguard_received_bytes_total[5m])))/(sum(rate(wireguard_sent_bytes_total[5m])) + sum(rate(wireguard_received_bytes_total[5m])))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Slip (send - received)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:502",
+          "format": "percentunit",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:503",
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 16,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum by (public_key) (time() - (wireguard_latest_handshake_seconds!=0))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "latest handshake",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:219",
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": "1000",
+          "min": "0",
+          "show": true
+        },
+        {
+          "$$hashKey": "object:220",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 18,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (instance) (rate(kilo_reconciles_total[30m]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "kilo reconciles",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:539",
+          "decimals": null,
+          "format": "hertz",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:540",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 0,
+        "y": 16
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.5.4",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "avg(kilo_peers)",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "title": "Kilo Peers",
+      "type": "stat"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 4,
+        "y": 16
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.5.4",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "avg(kilo_nodes)",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Kilo Nodes",
+      "type": "stat"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 8,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.5.4",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum(kilo_leader)",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "title": "segments",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum by (instance) (rate(kilo_errors_total[10m]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Kilo Errors",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:446",
+          "format": "hertz",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:447",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "hiddenSeries": false,
+      "id": 20,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (instance) (rate(process_cpu_seconds_total{pod=~\"kilo-.*\"}[1m]))",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "CPU usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:162",
+          "format": "percentunit",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:163",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "hiddenSeries": false,
+      "id": 22,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.5.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "sum by (instance) (process_resident_memory_bytes{pod=~\"kilo-.*\"})",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Memory Allocation",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:231",
+          "format": "decbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:232",
+          "format": "decmbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 32
+      },
+      "id": 14,
+      "panels": [],
+      "title": "Row title",
+      "type": "row"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Kilo",
+  "uid": "R8Lja3H7z",
+  "version": 11
+}

docs/kg.md

@@ -0,0 +1,60 @@
+# kg
+
+`kg` is the Kilo agent that runs on every Kubernetes node in a Kilo mesh.
+It performs several key functions, including:
+* adding the node to the Kilo mesh;
+* installing CNI configuration on the node;
+* configuring the WireGuard network interface; and
+* maintaining routing table entries and iptables rules.
+
+`kg` is typically installed on all nodes of a Kubernetes cluster using a DaemonSet.
+Example manifests can be found [in the manifests directory](https://github.com/squat/kilo/tree/main/manifests).
+
+## Usage
+
+The behavior of `kg` can be configured using the command line flags listed below.
+
+[embedmd]:# (../tmp/help.txt)
+```txt
+kg is the Kilo agent.
+		It runs on every node of a cluster,
+		setting up the public and private keys for the VPN
+		as well as the necessary rules to route packets between locations.
+
+Usage:
+  kg [flags]
+  kg [command]
+
+Available Commands:
+  completion  generate the autocompletion script for the specified shell
+  help        Help about any command
+  version     Print the version and exit.
+  webhook     webhook starts a HTTPS server to validate updates and creations of Kilo peers.
+
+Flags:
+      --backend string                 The backend for the mesh. Possible values: kubernetes (default "kubernetes")
+      --clean-up-interface             Should Kilo delete its interface when it shuts down?
+      --cni                            Should Kilo manage the node's CNI configuration? (default true)
+      --cni-path string                Path to CNI config. (default "/etc/cni/net.d/10-kilo.conflist")
+      --compatibility string           Should Kilo run in compatibility mode? Possible values: flannel
+      --create-interface               Should kilo create an interface on startup? (default true)
+      --encapsulate string             When should Kilo encapsulate packets within a location? Possible values: never, crosssubnet, always (default "always")
+  -h, --help                           help for kg
+      --hostname string                Hostname of the node on which this process is running.
+      --interface string               Name of the Kilo interface to use; if it does not exist, it will be created. (default "kilo0")
+      --iptables-forward-rules         Add default accept rules to the FORWARD chain in iptables. Warning: this may break firewalls with a deny all policy and is potentially insecure!
+      --kubeconfig string              Path to kubeconfig.
+      --listen string                  The address at which to listen for health and metrics. (default ":1107")
+      --local                          Should Kilo manage routes within a location? (default true)
+      --log-level string               Log level to use. Possible values: all, debug, info, warn, error, none (default "info")
+      --master string                  The address of the Kubernetes API server (overrides any value in kubeconfig).
+      --mesh-granularity string        The granularity of the network mesh to create. Possible values: location, full (default "location")
+      --mtu uint                       The MTU of the WireGuard interface created by Kilo. (default 1420)
+      --port int                       The port over which WireGuard peers should communicate. (default 51820)
+      --prioritise-private-addresses   Prefer to assign a private IP address to the node's endpoint.
+      --resync-period duration         How often should the Kilo controllers reconcile? (default 30s)
+      --subnet string                  CIDR from which to allocate addresses for WireGuard interfaces. (default "10.4.0.0/16")
+      --topology-label string          Kubernetes node label used to group nodes into logical locations. (default "topology.kubernetes.io/region")
+      --version                        Print version and exit
+
+```

docs/kgctl.md

@@ -6,21 +6,95 @@ This tool can be used to understand a mesh's topology, get the WireGuard configu
 
 ## Installation
 
-Installing `kgctl` currently requires building the binary from source.
+The `kgctl` binary is automatically compiled for Linux, macOS, and Windows for every release of Kilo and can be downloaded from [the GitHub releases page](https://github.com/squat/kilo/releases/latest).
-*Note*: the [Go toolchain must be installed](https://golang.org/doc/install) in order to build the binary.
+
-To build and install `kgctl`, simply run:
+### Building from Source
+Kilo is written in Golang and as a result the [Go toolchain must be installed](https://golang.org/doc/install) in order to build the `kgctl` binary.
+To download the Kilo source code and then build and install `kgctl` using the latest commit all with a single command, run:
 
 ```shell
-go install github.com/squat/kilo/cmd/kgctl
+go install github.com/squat/kilo/cmd/kgctl@latest
+```
+
+Alternatively, `kgctl` can be built and installed based on specific version of the code by specifying a Git tag or hash, e.g.:
+
+```shell
+go install github.com/squat/kilo/cmd/kgctl@0.2.0
+```
+
+When working on Kilo locally, it can be helpful to build and test the `kgctl` binary as part of the development cycle.
+In order to build a binary from a local checkout of the Git repository, run:
+
+```shell
+make
+```
+
+This will produce a `kgctl` binary at `./bin/<your-os>/<your-architecture>/kgctl`.
+
+
+### Binary Packages
+
+#### Arch Linux
+
+Install `kgctl` from the Arch User Repository using an AUR helper like `paru` or `yay`:
+
+```shell
+paru -S kgctl-bin
+```
+
+#### Arkade
+
+The [arkade](https://github.com/alexellis/arkade) CLI can be used to install `kgctl` on any OS and architecture:
+
+```shell
+arkade get kgctl
 ```
 
 ## Commands
 
 |Command|Syntax|Description|
 |----|----|-------|
+|[connect](#connect)|`kgctl connect <peer-name> [flags]`|Connect the host to the cluster, setting up the required interfaces, routes, and keys.|
 |[graph](#graph)|`kgctl graph [flags]`|Produce a graph in GraphViz format representing the topology of the cluster.|
-|[showconf](#showconf)|`kgctl showconf ( node \| peer ) NAME [flags]`|Show the WireGuard configuration for a node or peer in the mesh.|
+|[showconf](#showconf)|`kgctl showconf ( node \| peer ) <name> [flags]`|Show the WireGuard configuration for a node or peer in the mesh.|
 
+### connect
+
+The `connect` command configures the local host as a WireGuard Peer of the cluster and applies all of the necessary networking configuration to connect to the cluster.
+As long as the process is running, it will watch the cluster for changes and automatically manage the configuration for new or updated Peers and Nodes.
+If the given Peer name does not exist in the cluster, the command will register a new Peer and generate the necessary WireGuard keys.
+When the command exits, all of the configuration, including newly registered Peers, is cleaned up.
+
+Example:
+
+```shell
+PEER_NAME=laptop
+SERVICECIDR=10.43.0.0/16
+kgctl connect $PEER_NAME --allowed-ips $SERVICECIDR
+```
+
+The local host is now connected to the cluster and all IPs from the cluster and any registered Peers are fully routable.
+When combined with the `--clean-up false` flag, the configuration produced by the command is persistent and will remain in effect even after the process is stopped.
+
+With the service CIDR of the cluster routable from the local host, Kubernetes DNS names can now be resolved by the cluster DNS provider.
+For example, the following snippet could be used to resolve the clusterIP of the Kubernetes API:
+```shell
+dig @$(kubectl get service -n kube-system kube-dns -o=jsonpath='{.spec.clusterIP}') kubernetes.default.svc.cluster.local +short
+# > 10.43.0.1
+```
+
+For convenience, the cluster DNS provider's IP address can be configured as the local host's DNS server, making Kubernetes DNS names easily resolvable.
+For example, if using `systemd-resolved`, the following snippet could be used:
+```shell
+systemd-resolve --interface kilo0 --set-dns $(kubectl get service -n kube-system kube-dns -o=jsonpath='{.spec.clusterIP}') --set-domain cluster.local
+# Now all lookups for DNS names ending in `.cluster.local` will be routed over the `kilo0` interface to the cluster DNS provider.
+dig kubernetes.default.svc.cluster.local +short
+# > 10.43.0.1
+```
+
+> **Note**: The `connect` command is currently only supported on Linux.
+
+> **Note**: The `connect` command requires the `CAP_NET_ADMIN` capability in order to configure the host's networking stack; unprivileged users will need to use `sudo` or similar tools.
 
 ### graph
 

docs/monitoring.md

@@ -0,0 +1,100 @@
+# Monitoring
+
+The following assumes that you have applied the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) monitoring stack onto your cluster.
+
+## Kilo
+
+Monitor the Kilo DaemonSet with:
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/podmonitor.yaml
+```
+
+## WireGuard
+
+Monitor the WireGuard interfaces with:
+```shell
+kubectl create ns kilo
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/wg-exporter.yaml
+```
+
+The manifest will deploy the [Prometheus WireGuard Exporter](https://github.com/MindFlavor/prometheus_wireguard_exporter) as a DaemonSet and a [PodMonitor](https://docs.openshift.com/container-platform/4.8/rest_api/monitoring_apis/podmonitor-monitoring-coreos-com-v1.html).
+
+By default the kube-prometheus stack only monitors the `default`, `kube-system` and `monitoring` namespaces.
+In order to allow Prometheus to monitor the `kilo` namespace, apply the Role and RoleBinding with:
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/wg-exporter-role-kube-prometheus.yaml
+```
+
+## Metrics
+
+### Kilo
+
+Kilo exports some standard metrics with the Prometheus GoCollector and ProcessCollector.
+It also exposes some Kilo-specific metrics.
+
+```
+# HELP kilo_errors_total Number of errors that occurred while administering the mesh.
+# TYPE kilo_errors_total counter
+
+# HELP kilo_leader Leadership status of the node.
+# TYPE kilo_leader gauge
+
+# HELP kilo_nodes Number of nodes in the mesh.
+# TYPE kilo_nodes gauge
+
+# HELP kilo_peers Number of peers in the mesh.
+# TYPE kilo_peers gauge
+
+# HELP kilo_reconciles_total Number of reconciliation attempts.
+# TYPE kilo_reconciles_total counter
+```
+
+### WireGuard
+
+The [Prometheus WireGuard Exporter](https://github.com/MindFlavor/prometheus_wireguard_exporter) exports the following metrics:
+
+```
+# HELP wireguard_sent_bytes_total Bytes sent to the peer
+# TYPE wireguard_sent_bytes_total counter
+
+# HELP wireguard_received_bytes_total Bytes received from the peer
+# TYPE wireguard_received_bytes_total counter
+
+# HELP wireguard_latest_handshake_seconds Seconds from the last handshake
+# TYPE wireguard_latest_handshake_seconds gauge
+```
+
+## Display some Metrics
+
+If your laptop is a Kilo peer of the cluster you can access the Prometheus UI by navigating your browser directly to the cluster IP of the `prometheus-k8s` service.
+Otherwise use `port-forward`:
+```shell
+kubectl -n monitoring port-forward svc/prometheus-k8s 9090
+```
+and navigate your browser to `localhost:9090`.
+Check if you can see the PodMonitors for Kilo and the WireGuard Exporter under **Status** -> **Targets** in the Prometheus web UI.
+
+If you don't see them, check the logs of the `prometheus-k8s` Pods; it may be that Prometheus doesn't have the permission to get Pods in the `kilo` namespace.
+In this case, you need to apply the Role and RoleBinding from above.
+
+Navigate to **Graph** and try to execute a simple query, e.g. type `kilo_nodes` and click on `execute`.
+You should see some data.
+
+## Using Grafana
+
+Let's navigate to the Grafana dashboard.
+Again, if your laptop is not a Kilo peer, use `port-forward`:
+```shell
+kubectl -n monitoring port-forward svc/grafana 3000
+```
+
+Now navigate your browser to `localhost:3000`.
+The default user and password is `admin` `admin`.
+
+An example configuration for a dashboard displaying Kilo metrics can be found [here](https://raw.githubusercontent.com/squat/kilo/main/docs/grafana/kilo.json).
+You can import this dashboard by hitting **+** -> **Import** on the Grafana dashboard.
+
+The dashboard looks like this:
+
+<img src="./graphs/kilo.png" />
+

docs/multi-cluster-services.md

@@ -5,7 +5,7 @@ This enables clusters to provide services to other clusters over a secure connec
 For example, a cluster on AWS with access to GPUs could run a machine learning service that could be consumed by workloads running in a another location, e.g. an on-prem cluster without GPUs.
 Unlike services exposed via Ingresses or NodePort Services, multi-cluster services can remain private and internal to the clusters.
 
-*Note*: in order for connected clusters to be fully routable, the allowed IPs that they declare must be non-overlapping, i.e. the Kilo, pod, and service CIDRs.
+> **Note**: in order for connected clusters to be fully routable, the allowed IPs that they declare must be non-overlapping, i.e. the Kilo, pod, and service CIDRs.
 
 ## Getting Started
 

docs/network-policies.md

@@ -10,7 +10,7 @@ Support for [Kubernetes network policies](https://kubernetes.io/docs/concepts/se
 The following command adds network policy support by deploying kube-router to work alongside Kilo:
 
 ```shell
-kubectl apply -f kubectl apply -f https://raw.githubusercontent.com/squat/kilo/master/manifests/kube-router.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kube-router.yaml
 ```
 
 ## Examples

docs/peer-validation.md

@@ -0,0 +1,22 @@
+# Peer Validation
+
+A [ValidatingAdmissionWebhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook) can  be used to avoid applying faulty Peer configurations to the cluster.
+
+## How It Works
+
+A [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly) is a Kubernetes resource that can be used to dynamically specify a service (i.e. the webhook server) that should validate operations (e.g. `UPDATE`, `CREATE`, etc.) on a particular resource (e.g. Kilo Peers).
+Once such a configuration is applied, the Kubernetes API server will send an AdmissionReviewRequest to the webhook service every time the specified operations are applied to the resource of the specified type.
+With regard to the [failure policy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy), the API server will apply the requested changes to a resource if the request was answered with `"allowed": true`, or deny the changes if the answer was `"allowed": false`.
+
+In case of Kilo Peer Validation, the specified operations are `UPDATE` and `CREATE`, the resources are `Peers`, and the default `failurePolicy` is set to `Fail`.
+View the full ValidatingWebhookConfiguration [here](https://github.com/squat/kilo/blob/main/manifests/peer-validation.yaml).
+
+## Getting Started
+
+Apply the Service, the Deployment of the actual webserver, and the ValidatingWebhookConfiguration with:
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/blob/main/manifests/peer-validation.yaml
+```
+
+The Kubernetes API server will only talk to webhook servers via TLS so the Kilo-Peer-Validation server must be given a valid TLS certificate and key, and the API server must be told what certificate authority (CA) to trust.
+The above manifest will use [kube-webhook-certgen](https://github.com/jet/kube-webhook-certgen) to generate the requiered certificates and patch the [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly).

docs/topology.md

@@ -11,9 +11,10 @@ This allows the encrypted network to serve several purposes, for example:
 
 By default, Kilo creates a mesh between the different logical locations in the cluster, e.g. data-centers, cloud providers, etc.
 Kilo will try to infer the location of the node using the [topology.kubernetes.io/region](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#topologykubernetesioregion) node label.
+Additionally, Kilo supports using a custom topology label by setting the command line flag `--topology-label=<label>`. 
 If this label is not set, then the [kilo.squat.ai/location](./annotations.md#location) node annotation can be used.
 
-For example, in order to join nodes in Google Cloud and AWS into a single cluster, an administrator could use the following snippet could to annotate all nodes with `GCP` in the name:
+For example, in order to join nodes in Google Cloud and AWS into a single cluster, an administrator could use the following snippet to annotate all nodes with `GCP` in the name:
 
 ```shell
 for node in $(kubectl get nodes | grep -i gcp | awk '{print $1}'); do kubectl annotate node $node kilo.squat.ai/location="gcp"; done

docs/userspace-wireguard.md

@@ -0,0 +1,44 @@
+# Userspace WireGuard
+
+It is possible to use a userspace implementation of WireGuard with Kilo.
+This can make sense in cases where
+
+* not all nodes in a cluster have WireGuard installed; or
+* nodes are effectively immutable and kernel modules cannot be installed.
+
+One example of a userspace implementation of WireGuard is [BoringTun].
+
+## Homogeneous Clusters
+
+In a homogeneous cluster where no node has the WireGuard kernel module, a userspace WireGuard implementation can be made available by deploying a DaemonSet.
+This DaemonSet creates a WireGuard interface that Kilo will manage.
+
+> **Note**: in order to avoid race conditions, `kg` needs to be passed the `--create-interface=false` flag. 
+
+An example configuration for a K3s cluster with [BoringTun] can be applied with:
+
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s-userspace.yaml
+```
+
+> **Note**: even if some nodes have the WireGuard kernel module, this configuration will cause all nodes to use the userspace implementation of WireGuard.
+
+## Heterogeneous Clusters
+
+In a heterogeneous cluster where some nodes are missing the WireGuard kernel module, a userspace WireGuard implementation can be provided only to the nodes that need it while enabling the other nodes to leverage WireGuard via the kernel module.
+An example of such a configuration for a K3s cluster can by applied with:
+
+```shell
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/crds.yaml
+kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/kilo-k3s-userspace-heterogeneous.yaml
+```
+
+This configuration will deploy [nkml](https://github.com/leonnicolas/nkml) as a DaemonSet to label all nodes according to the presence of the WireGuard kernel module.
+It will also create two different DaemonSets with Kilo:
+1. `kilo` without userspace WireGuard; and
+1. `kilo-userspace` with [BoringTun] as a sidecar.
+
+> **Note**: because Kilo is dependant on nkml, nkml must be run on the host network before CNI is available and requires a kubeconfig in order to access the Kubernetes API.
+
+[BoringTun]: https://github.com/cloudflare/boringtun

docs/vpn.md

@@ -3,6 +3,7 @@
 Kilo enables peers outside of a Kubernetes cluster to connect to the created WireGuard network.
 This enables several use cases, for example:
 * giving cluster applications secure access to external services, e.g. services behind a corporate VPN;
+* improving the development flow of applications by running them locally and connecting them to the cluster;
 * allowing external services to access the cluster; and
 * enabling developers and support to securely debug cluster resources.
 
@@ -74,7 +75,7 @@ From any node or Pod on the cluster, one can now ping the peer:
 ping 10.5.0.1
 ```
 
-If the peer exposes a layer 4 service, for example an HTTP service, then one could also make requests against that endpoint from the cluster:
+If the peer exposes a layer 4 service, for example an HTTP server listening on TCP port 80, then one could also make requests against that endpoint from the cluster:
 
 ```shell
 curl http://10.5.0.1

e2e/full-mesh.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_full_mesh_connectivity() {
+	assert "retry 30 5 '' check_ping" "should be able to ping all Pods"
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings"
+	echo "sleep for 30s (one reconciliation period) and try again..."
+	sleep 30
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling"
+}
+
+test_full_mesh_peer() {
+	check_peer wg99 e2e 10.5.0.1/32 full
+}
+
+test_full_mesh_allowed_location_ips() {
+	docker exec kind-cluster-kilo-control-plane ip address add 10.6.0.1/32 dev eth0
+	_kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips=10.6.0.1/32
+	assert_equals Unauthorized "$(retry 10 5 'IP is not yet routable' curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz)" "should be able to make HTTP request to allowed location IP"
+	_kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips-
+	assert "retry 10 5 'IP is still routable' _not curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz" "should not be able to make HTTP request to allowed location IP"
+	docker exec kind-cluster-kilo-control-plane ip address delete 10.6.0.1/32 dev eth0
+}
+
+test_reject_peer_empty_allowed_ips() {
+	assert_fail "create_peer e2e '' 0 foo" "should not be able to create Peer with empty allowed IPs"
+}
+
+test_reject_peer_empty_public_key() {
+	assert_fail "create_peer e2e 10.5.0.1/32 0 ''" "should not be able to create Peer with empty public key"
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity full)"
+}

e2e/handlers.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_graph_handler() {
+    assert "curl_pod 'http://10.4.0.1:1107/graph?format=svg&layout=circo' | grep -q '<svg'" "graph handler should produce SVG output"
+    assert "curl_pod http://10.4.0.1:1107/graph?layout=circo | grep -q '<svg'" "graph handler should default to SVG output"
+    assert "curl_pod http://10.4.0.1:1107/graph | grep -q '<svg'" "graph handler should default to SVG output"
+    assert_fail "curl_pod http://10.4.0.1:1107/graph?layout=fake | grep -q '<svg'" "graph handler should reject invalid layout"
+    assert_fail "curl_pod http://10.4.0.1:1107/graph?format=fake | grep -q '<svg'" "graph handler should reject invalid format"
+}
+
+test_health_handler() {
+    assert "curl_pod http://10.4.0.1:1107/health" "health handler should return a status code of 200"
+}
+
+test_metrics_handler() {
+    assert "curl_pod http://10.4.0.1:1107/metrics" "metrics handler should return a status code of 200"
+    assert "(( $(curl_pod http://10.4.0.1:1107/metrics | grep -E ^kilo_nodes | cut -d " " -f 2) > 0 ))" "metrics handler should provide metric: kilo_nodes > 0"
+}

e2e/helper-curl.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+  labels:
+    app.kubernetes.io/name: curl
+  name: curl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: curl
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: curl
+    spec:
+      containers:
+      - command:
+        - /bin/sh
+        - -c
+        - while [ 1 -eq 1 ] ; do sleep 10; done
+        image: curlimages/curl
+        name: curl
+      restartPolicy: Always

e2e/kgctl.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_connect() {
+	local PEER=test
+	local ALLOWED_IP=10.5.0.1/32
+        docker run -d --name="$PEER" --rm --network=host --cap-add=NET_ADMIN -v "$KGCTL_BINARY":/kgctl -v "$PWD/$KUBECONFIG":/kubeconfig --entrypoint=/kgctl alpine --kubeconfig /kubeconfig connect "$PEER" --allowed-ip "$ALLOWED_IP"
+	assert "retry 10 5 '' check_ping --local" "should be able to ping Pods from host"
+        docker stop "$PEER"
+}

e2e/kilo-kind-userspace.yaml

@@ -0,0 +1,203 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.4.0",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:test
+        imagePullPolicy: Never
+        args:
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --mesh-granularity=full
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+      - name: boringtun
+        image: leonnicolas/boringtun
+        args:
+        - --disable-drop-privileges
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo:test
+        imagePullPolicy: Never
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig

e2e/kind-config.yaml

@@ -0,0 +1,10 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane$WORKERS
+networking:
+  disableDefaultCNI: true # disable kindnet
+  apiServerAddress: 172.18.0.1
+  apiServerPort: $API_SERVER_PORT
+  podSubnet: $POD_SUBNET
+  serviceSubnet: $SERVICE_SUBNET

e2e/lib.sh

@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+KUBECONFIG="kind.yaml"
+KIND_CLUSTER="kind-cluster-kilo"
+KIND_BINARY="${KIND_BINARY:-kind}"
+KUBECTL_BINARY="${KUBECTL_BINARY:-kubectl}"
+KGCTL_BINARY="${KGCTL_BINARY:-kgctl}"
+KILO_IMAGE="${KILO_IMAGE:-squat/kilo}"
+
+retry() {
+	local COUNT="${1:-10}"
+	local SLEEP="${2:-5}"
+	local ERROR=$3
+	[ -n "$ERROR" ] && ERROR="$ERROR "
+	shift 3
+	for c in $(seq 1 "$COUNT"); do
+		if "$@"; then
+			return 0
+		else
+			printf "%s(attempt %d/%d)\n" "$ERROR" "$c" "$COUNT" | color "$YELLOW" 1>&2
+			if [ "$c" != "$COUNT" ]; then
+				printf "retrying in %d seconds...\n" "$SLEEP" | color "$YELLOW" 1>&2
+				sleep "$SLEEP"
+			fi
+		fi
+	done
+	return 1
+}
+
+_not() {
+	if "$@"; then
+		return 1
+	fi
+	return 0
+}
+
+# _kubectl is a helper that calls kubectl with the --kubeconfig flag.
+_kubectl() {
+	$KUBECTL_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# _kgctl is a helper that calls kgctl with the --kubeconfig flag.
+_kgctl() {
+	$KGCTL_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# _kind is a helper that calls kind with the --kubeconfig flag.
+_kind() {
+	$KIND_BINARY --kubeconfig="$KUBECONFIG" "$@"
+}
+
+# shellcheck disable=SC2120
+build_kind_config() {
+	local WORKER_COUNT="${1:-0}"
+	export API_SERVER_PORT="${2:-6443}"
+	export POD_SUBNET="${3:-10.42.0.0/16}"
+	export SERVICE_SUBNET="${4:-10.43.0.0/16}"
+	export WORKERS=""
+	local i=0
+	while [ "$i" -lt "$WORKER_COUNT" ]; do
+		WORKERS="$(printf "%s\n- role: worker" "$WORKERS")"
+		((i++))
+	done
+	envsubst < ./kind-config.yaml
+	unset API_SERVER_PORT POD_SUBNET SERVICE_SUBNET WORKERS
+}
+
+create_interface() {
+	docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun --foreground --disable-drop-privileges "$1"
+}
+
+delete_interface() {
+	docker rm --force "$1"
+}
+
+create_peer() {
+	cat <<EOF | _kubectl apply -f -
+apiVersion: kilo.squat.ai/v1alpha1
+kind: Peer
+metadata:
+  name: $1
+spec:
+  allowedIPs:
+  - $2
+  persistentKeepalive: $3
+  publicKey: $4
+EOF
+}
+
+delete_peer() {
+	_kubectl delete peer "$1"
+}
+
+is_ready() {
+	for pod in $(_kubectl -n "$1" get pods -o name -l "$2"); do
+		if ! _kubectl -n "$1" get "$pod" | tail -n 1 | grep -q Running; then
+			return 1;
+		fi
+	done
+	return 0
+}
+
+# Returns non zero if one pod of the given name in the given namespace is not ready.
+block_until_ready_by_name() {
+	block_until_ready "$1" "app.kubernetes.io/name=$2"
+}
+
+# Blocks until all pods of a deployment are ready.
+block_until_ready() {
+	retry 30 5 "some $2 pods are not ready yet" is_ready "$1" "$2"
+}
+
+
+# create_cluster launches a kind cluster and deploys Kilo, Adjacency, and a helper with curl.
+create_cluster() {
+	# shellcheck disable=SC2119
+	local CONFIG="${1:-$(build_kind_config)}"
+	_kind delete clusters $KIND_CLUSTER > /dev/null
+	# Create the kind cluster.
+	_kind create cluster --name $KIND_CLUSTER --config <(echo "$CONFIG")
+	# Load the Kilo image into kind.
+	docker tag "$KILO_IMAGE" squat/kilo:test
+	# This command does not accept the --kubeconfig flag, so call the command directly.
+	$KIND_BINARY load docker-image squat/kilo:test --name $KIND_CLUSTER
+	# Create the kubeconfig secret.
+	_kubectl create secret generic kubeconfig --from-file=kubeconfig="$KUBECONFIG" -n kube-system
+	# Apply Kilo the the cluster.
+	_kubectl apply -f ../manifests/crds.yaml
+	_kubectl apply -f kilo-kind-userspace.yaml
+	block_until_ready_by_name kube-system kilo-userspace
+	_kubectl wait nodes --all --for=condition=Ready
+	# Wait for CoreDNS.
+	block_until_ready kube_system k8s-app=kube-dns
+	# Ensure the curl helper is not scheduled on a control-plane node.
+	_kubectl apply -f helper-curl.yaml
+	block_until_ready_by_name default curl
+	_kubectl taint node $KIND_CLUSTER-control-plane node-role.kubernetes.io/master:NoSchedule-
+	_kubectl apply -f https://raw.githubusercontent.com/kilo-io/adjacency/main/example.yaml
+	block_until_ready_by_name default adjacency
+}
+
+delete_cluster () {
+	_kind delete clusters $KIND_CLUSTER
+}
+
+curl_pod() {
+	_kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /usr/bin/curl "$@"
+}
+
+check_ping() {
+	local LOCAL
+	while [ $# -gt 0 ]; do
+		case $1 in
+			--local)
+			LOCAL=true
+			;;
+		esac
+		shift
+	done
+
+	for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		if [ -n "$LOCAL" ]; then
+			ping=$(curl -m 1 -s http://"$ip":8080/ping)
+		else
+			ping=$(curl_pod -m 1 -s http://"$ip":8080/ping)
+		fi
+		if [ "$ping" = "pong" ]; then
+			echo "successfully pinged $ip"
+		else
+			printf 'failed to ping %s; expected "pong" but got "%s"\n' "$ip" "$ping"
+			return 1
+		fi
+	done
+	return 0
+}
+
+check_adjacent() {
+	curl_pod adjacency:8080/?format=fancy
+	[ "$(curl_pod -m 1 -s adjacency:8080/?format=json | jq '.[].latencies[].ok' | grep -c true)" -eq $(($1*$1)) ]
+}
+
+check_peer() {
+	local INTERFACE=$1
+	local PEER=$2
+	local ALLOWED_IP=$3
+	local GRANULARITY=$4
+	create_interface "$INTERFACE"
+	docker run --rm leonnicolas/wg-tools wg genkey > "$INTERFACE"
+	assert "create_peer $PEER $ALLOWED_IP 10 $(docker run --rm --entrypoint=/bin/sh -v "$PWD/$INTERFACE":/key leonnicolas/wg-tools -c 'cat /key | wg pubkey')" "should be able to create Peer"
+	assert "_kgctl showconf peer $PEER --mesh-granularity=$GRANULARITY > $PEER.ini" "should be able to get Peer configuration"
+	assert "docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v $PWD/$PEER.ini:/peer.ini leonnicolas/wg-tools setconf $INTERFACE /peer.ini" "should be able to apply configuration from kgctl"
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v "$PWD/$INTERFACE":/key leonnicolas/wg-tools set "$INTERFACE" private-key /key
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip leonnicolas/wg-tools address add "$ALLOWED_IP" dev "$INTERFACE"
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip leonnicolas/wg-tools link set "$INTERFACE" up
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip leonnicolas/wg-tools route add 10.42/16 dev "$INTERFACE"
+	assert "retry 10 5 '' check_ping --local" "should be able to ping Pods from host"
+	assert_equals "$(_kgctl showconf peer "$PEER")" "$(_kgctl showconf peer "$PEER" --mesh-granularity="$GRANULARITY")" "kgctl should be able to auto detect the mesh granularity"
+	rm "$INTERFACE" "$PEER".ini
+	delete_peer "$PEER"
+	delete_interface "$INTERFACE"
+}

e2e/location-mesh.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace 
+	_kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
+}
+
+test_location_mesh_connectivity() {
+	assert "retry 30 5 '' check_ping" "should be able to ping all Pods"
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings"
+	echo "sleep for 30s (one reconciliation period) and try again..."
+	sleep 30
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling"
+}
+
+test_location_mesh_peer() {
+	check_peer wg99 e2e 10.5.0.1/32 location
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity location)"
+}

e2e/multi-cluster.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+# shellcheck disable=SC2153
+KUBECONFIG2="$KUBECONFIG"2
+# shellcheck disable=SC2153
+KIND_CLUSTER2="$KIND_CLUSTER"2
+
+setup_suite() {
+	KUBECONFIG=$KUBECONFIG2 KIND_CLUSTER=$KIND_CLUSTER2 create_cluster "$(build_kind_config 1 6444 10.44.0.0/16 10.45.0.0/16)"
+	# shellcheck disable=SC2016
+	KUBECONFIG=$KUBECONFIG2 _kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full","--subnet=10.6.0.0/16"]}]}}}}'
+	KUBECONFIG=$KUBECONFIG2 block_until_ready_by_name kube-system kilo-userspace 
+	# Register the nodes in cluster1 as peers of cluster2.
+	for n in $(_kubectl get no -o name | cut -d'/' -f2); do
+		# Specify the service CIDR as an extra IP range that should be routable.
+		$KGCTL_BINARY --kubeconfig "$KUBECONFIG" showconf node "$n" --as-peer -o yaml --allowed-ips 10.43.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG2" apply -f -
+	done
+	# Register the nodes in cluster2 as peers of cluster1.
+	for n in $(KUBECONFIG=$KUBECONFIG2 _kubectl get no -o name | cut -d'/' -f2); do
+		# Specify the service CIDR as an extra IP range that should be routable.
+		$KGCTL_BINARY --kubeconfig "$KUBECONFIG2" showconf node "$n" --as-peer -o yaml --allowed-ips 10.45.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG" apply -f -
+	done
+}
+
+test_multi_cluster_pod_connectivity() {
+	for ip in $(KUBECONFIG=$KUBECONFIG2 _kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		assert_equals pong "$(retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 1 to Pod in cluster 2"
+	done
+	for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		assert_equals pong "$(KUBECONFIG="$KUBECONFIG2" retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 2 to Pod in cluster 1"
+	done
+}
+
+test_multi_cluster_service_connectivity() {
+	# Mirror the Kubernetes API service from cluster1 into cluster2.
+	cat <<EOF | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG2" apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: mirrored-kubernetes
+spec:
+  ports:
+    - port: 443
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+    name: mirrored-kubernetes
+subsets:
+  - addresses:
+      - ip: $(_kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}') # The cluster IP of the Kubernetes API service on cluster1.
+    ports:
+      - port: 443
+EOF
+        assert_equals ok "$(KUBECONFIG="$KUBECONFIG2" retry 10 5 "service is not yet routable" curl_pod -m 1 -s -k https://mirrored-kubernetes/readyz)" "should be able to make HTTP request from cluster 2 to service in cluster 1"
+}
+
+teardown_suite () {
+	# Remove the nodes in cluster2 as peers of cluster1.
+	for n in $(KUBECONFIG=$KUBECONFIG2 _kubectl get no -o name | cut -d'/' -f2); do
+		_kubectl delete peer "$n"
+	done
+	KUBECONFIG=$KUBECONFIG2 KIND_CLUSTER=$KIND_CLUSTER2 delete_cluster
+}

e2e/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+setup_suite() {
+	create_cluster "$(build_kind_config 2)"
+}

e2e/teardown.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+teardown_suite () {
+	delete_cluster
+}

go.mod

@@ -1,64 +1,88 @@
 module github.com/squat/kilo
 
-go 1.14
+go 1.18
 
 require (
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/ant31/crd-validation v0.0.0-20180801212718-38f6a293f140
 	github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310
-	github.com/containernetworking/cni v0.6.0
+	github.com/campoy/embedmd v1.0.0
-	github.com/containernetworking/plugins v0.6.0
+	github.com/containernetworking/cni v1.0.1
-	github.com/coreos/go-iptables v0.4.0
+	github.com/containernetworking/plugins v1.1.1
-	github.com/emicklei/go-restful v2.9.3+incompatible // indirect
+	github.com/coreos/go-iptables v0.6.0
-	github.com/evanphx/json-patch v4.2.0+incompatible // indirect
+	github.com/go-kit/kit v0.9.0
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-kit/kit v0.8.0
-	github.com/go-logfmt/logfmt v0.4.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.0 // indirect
-	github.com/go-openapi/jsonreference v0.19.0 // indirect
-	github.com/go-openapi/spec v0.19.0
-	github.com/go-openapi/swag v0.19.0 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect
-	github.com/golang/protobuf v1.3.1 // indirect
-	github.com/google/gofuzz v1.0.0 // indirect
-	github.com/googleapis/gnostic v0.2.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
-	github.com/json-iterator/go v1.1.6 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
-	github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 // indirect
+	github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/oklog/run v1.1.0
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/prometheus/client_golang v1.11.0
-	github.com/oklog/run v1.0.0
+	github.com/spf13/cobra v1.2.1
-	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
-	github.com/onsi/gomega v1.5.0 // indirect
+	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881
-	github.com/prometheus/client_golang v0.9.2
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211124212657-dd7407c86d22
-	github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267
+	honnef.co/go/tools v0.3.1
-	github.com/stretchr/testify v1.3.0 // indirect
+	k8s.io/api v0.23.6
-	github.com/vishvananda/netlink v1.0.0
+	k8s.io/apiextensions-apiserver v0.23.6
-	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
+	k8s.io/apimachinery v0.23.6
-	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
+	k8s.io/client-go v0.23.6
-	golang.org/x/lint v0.0.0-20190409202823-959b441ac422
+	k8s.io/code-generator v0.23.6
-	golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 // indirect
+	sigs.k8s.io/controller-tools v0.8.0
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
+)
-	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872
+
-	golang.org/x/text v0.3.2 // indirect
+require (
-	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
+	github.com/BurntSushi/toml v0.4.1 // indirect
-	golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
-	google.golang.org/appengine v1.5.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
-	k8s.io/api v0.0.0-20190313235455-40a48860b5ab
+	github.com/fatih/color v1.12.0 // indirect
-	k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed
+	github.com/go-logfmt/logfmt v0.5.0 // indirect
-	k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1
+	github.com/go-logr/logr v1.2.0 // indirect
-	k8s.io/client-go v11.0.0+incompatible
+	github.com/gobuffalo/flect v0.2.3 // indirect
-	k8s.io/code-generator v0.0.0-20190311093542-50b561225d70
+	github.com/gogo/protobuf v1.3.2 // indirect
-	k8s.io/gengo v0.0.0-20181106084056-51747d6e00da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
-	k8s.io/klog v0.3.0 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
-	k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30
+	github.com/google/gofuzz v1.1.0 // indirect
-	k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 // indirect
+	github.com/google/uuid v1.2.0 // indirect
-	sigs.k8s.io/yaml v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mdlayher/genetlink v1.0.0 // indirect
+	github.com/mdlayher/netlink v1.4.1 // indirect
+	github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.28.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
+	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20211123210315-387f7c461a16 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c // indirect
+	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

manifests/crds.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+  creationTimestamp: null
+  name: peers.kilo.squat.ai
+spec:
+  group: kilo.squat.ai
+  names:
+    kind: Peer
+    listKind: PeerList
+    plural: peers
+    singular: peer
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Peer is a WireGuard peer that should have access to the VPN.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: 'Specification of the desired behavior of the Kilo Peer.
+              More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status'
+            properties:
+              allowedIPs:
+                description: AllowedIPs is the list of IP addresses that are allowed
+                  for the given peer's tunnel.
+                items:
+                  type: string
+                type: array
+              endpoint:
+                description: Endpoint is the initial endpoint for connections to the
+                  peer.
+                properties:
+                  dnsOrIP:
+                    description: DNSOrIP is a DNS name or an IP address.
+                    properties:
+                      dns:
+                        description: DNS must be a valid RFC 1123 subdomain.
+                        type: string
+                      ip:
+                        description: IP must be a valid IP address.
+                        type: string
+                    type: object
+                  port:
+                    description: Port must be a valid port number.
+                    format: int32
+                    type: integer
+                required:
+                - dnsOrIP
+                - port
+                type: object
+              persistentKeepalive:
+                description: PersistentKeepalive is the interval in seconds of the
+                  emission of keepalive packets by the peer. This defaults to 0, which
+                  disables the feature.
+                type: integer
+              presharedKey:
+                description: PresharedKey is the optional symmetric encryption key
+                  for the peer.
+                type: string
+              publicKey:
+                description: PublicKey is the WireGuard public key for the peer.
+                type: string
+            required:
+            - allowedIPs
+            - publicKey
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

manifests/kilo-bootkube-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,20 +51,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,6 +79,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-bootkube.yaml

@@ -8,7 +8,7 @@ metadata:
 data:
   cni-conf.json: |
     {
-       "cniVersion":"0.3.1",
+       "cniVersion":"0.4.0",
        "name":"kilo",
        "plugins":[
           {
@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,20 +85,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +110,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -126,7 +131,7 @@ spec:
           readOnly: false
       initContainers:
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         command:
         - /bin/sh
         - -c

manifests/kilo-k3s-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -45,6 +44,35 @@ subjects:
     name: kilo
     namespace: kube-system
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -52,20 +80,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,13 +108,16 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
         - name: kilo-dir
           mountPath: /var/lib/kilo
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
         - name: lib-modules
           mountPath: /lib/modules
@@ -91,6 +125,28 @@ spec:
         - name: xtables-lock
           mountPath: /run/xtables.lock
           readOnly: false
+      initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       tolerations:
       - effect: NoSchedule
         operator: Exists
@@ -101,11 +157,13 @@ spec:
         hostPath:
           path: /var/lib/kilo
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # Since kilo runs as a daemonset, it is recommended that you copy the
+          path: /var/lib/rancher/k3s/agent
-          # k3s.yaml kubeconfig file from the master node to all worker nodes
-          # with the same path structure.
-          path: /etc/rancher/k3s/k3s.yaml
       - name: lib-modules
         hostPath:
           path: /lib/modules

manifests/kilo-k3s-userspace-heterogeneous.yaml

@@ -0,0 +1,459 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.4.0",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - get
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      nodeSelector:
+        nkml.squat.ai/wireguard: "true"
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:0.5.0
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+      initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      - name: install-cni
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
+        hostPath:
+          path: /var/lib/rancher/k3s/agent
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo-userspace
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      nodeSelector:
+        nkml.squat.ai/wireguard: "false"
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:0.5.0
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      - name: boringtun
+        image: leonnicolas/boringtun:cc19859
+        args:
+        - --disable-drop-privileges
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      - name: install-cni
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
+        hostPath:
+          path: /var/lib/rancher/k3s/agent
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: nkml
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: nkml
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nkml
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nkml
+    spec:
+      hostNetwork: true
+      serviceAccountName: kilo
+      containers:
+      - name: nkml
+        image: leonnicolas/nkml
+        args:
+        - --hostname=$(NODE_NAME)
+        - --label-mod=wireguard
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - name: http
+          containerPort: 8080
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+      initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      volumes:
+      - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
+        hostPath:
+          path: /var/lib/rancher/k3s/agent

manifests/kilo-k3s-userspace.yaml

@@ -0,0 +1,257 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.4.0",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:0.5.0
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      - name: boringtun
+        image: leonnicolas/boringtun:cc19859
+        args:
+        - --disable-drop-privileges
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      - name: install-cni
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
+        hostPath:
+          path: /var/lib/rancher/k3s/agent
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard

manifests/kilo-k3s.yaml

@@ -8,7 +8,7 @@ metadata:
 data:
   cni-conf.json: |
     {
-       "cniVersion":"0.3.1",
+       "cniVersion":"0.4.0",
        "name":"kilo",
        "plugins":[
           {
@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -79,6 +78,35 @@ subjects:
     name: kilo
     namespace: kube-system
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo-scripts
+  namespace: kube-system
+data:
+  init.sh: |
+    #!/bin/sh
+    cat > /etc/kubernetes/kubeconfig <<EOF
+        apiVersion: v1
+        kind: Config
+        name: kilo
+        clusters:
+        - cluster:
+            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
+            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
+        users:
+        - name: kilo
+          user:
+            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+        contexts:
+        - name: kilo
+          context:
+            cluster: kilo
+            namespace: ${NAMESPACE}
+            user: kilo
+        current-context: kilo
+    EOF
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -86,20 +114,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +139,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -116,7 +150,7 @@ spec:
         - name: kilo-dir
           mountPath: /var/lib/kilo
         - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
+          mountPath: /etc/kubernetes
           readOnly: true
         - name: lib-modules
           mountPath: /lib/modules
@@ -125,8 +159,29 @@ spec:
           mountPath: /run/xtables.lock
           readOnly: false
       initContainers:
+      - name: generate-kubeconfig
+        image: squat/kilo:0.5.0
+        command:
+        - /bin/sh
+        args:
+        - /scripts/init.sh
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+        - name: scripts
+          mountPath: /scripts/
+          readOnly: true
+        - name: k3s-agent
+          mountPath: /var/lib/rancher/k3s/agent/
+          readOnly: true
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         command:
         - /bin/sh
         - -c
@@ -165,11 +220,13 @@ spec:
         hostPath:
           path: /var/lib/kilo
       - name: kubeconfig
+        emptyDir: {}
+      - name: scripts
+        configMap:
+          name: kilo-scripts
+      - name: k3s-agent
         hostPath:
-          # Since kilo runs as a daemonset, it is recommended that you copy the
+          path: /var/lib/rancher/k3s/agent
-          # k3s.yaml kubeconfig file from the master node to all worker nodes
-          # with the same path structure.
-          path: /etc/rancher/k3s/k3s.yaml
       - name: lib-modules
         hostPath:
           path: /lib/modules

manifests/kilo-kubeadm-cilium.yaml

@@ -0,0 +1,142 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:0.5.0
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --cni=false
+        - --compatibility=cilium
+        - --local=false
+        # additional and also optional flag
+        - --encapsulate=crosssubnet
+        - --clean-up-interface=true
+        - --subnet=172.31.254.0/24
+        - --log-level=all
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+
+        # with kube-proxy configmap
+        # - name: kubeconfig
+        #   mountPath: /etc/kubernetes
+        #   readOnly: true
+
+        # without kube-proxy host kubeconfig binding
+        - name: kubeconfig
+          mount_path: /etc/kubernetes/kubeconfig
+          sub_path: admin.conf
+          read_only: true
+
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+
+      # with kube-proxy configmap
+      # - name: kubeconfig
+      #   configMap:
+      #     name: kube-proxy
+      #     items:
+      #       - key: kubeconfig.conf
+      #         path: kubeconfig
+
+      # without kube-proxy host kubeconfig binding
+      - name: kubeconfig
+        host_path:
+          path: /etc/kubernetes
+
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate

manifests/kilo-kubeadm-flannel-userspace.yaml

@@ -0,0 +1,142 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: boringtun
+        image: leonnicolas/boringtun:cc19859
+        args:
+          - --disable-drop-privileges=true
+          - --foreground
+          - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: wireguard
+            mountPath: /var/run/wireguard
+            readOnly: false
+      - name: kilo
+        image: squat/kilo:0.5.0
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        - --cni=false
+        - --compatibility=flannel
+        - --local=false
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      tolerations:
+      - operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        configMap:
+          name: kube-proxy
+          items:
+          - key: kubeconfig.conf
+            path: kubeconfig
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard

manifests/kilo-kubeadm-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,20 +51,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,6 +79,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-kubeadm-userspace.yaml

@@ -0,0 +1,207 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.4.0",
+       "name":"kilo",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "forceAddress":true,
+             "mtu": 1420,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "snat":true,
+             "capabilities":{
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: boringtun
+        image: leonnicolas/boringtun:cc19859
+        imagePullPolicy: IfNotPresent
+        args:
+          - --disable-drop-privileges
+          - --foreground
+          - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: wireguard
+            mountPath: /var/run/wireguard
+            readOnly: false
+      - name: kilo
+        image: squat/kilo:0.5.0
+        imagePullPolicy: IfNotPresent
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --interface=kilo0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      initContainers:
+      - name: install-cni
+        image: squat/kilo:0.5.0
+        imagePullPolicy: IfNotPresent
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          cp /opt/cni/bin/* /host/opt/cni/bin/;
+          TMP_CONF="$CNI_CONF_NAME".tmp;
+          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
+          rm -f /host/etc/cni/net.d/*;
+          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
+        env:
+        - name: CNI_CONF_NAME
+          value: 10-kilo.conflist
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kilo
+              key: cni-conf.json
+        volumeMounts:
+        - name: cni-bin-dir
+          mountPath: /host/opt/cni/bin
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: kubeconfig
+        configMap:
+          name: kube-proxy
+          items:
+          - key: kubeconfig.conf
+            path: kubeconfig
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard

manifests/kilo-kubeadm.yaml

@@ -8,7 +8,7 @@ metadata:
 data:
   cni-conf.json: |
     {
-       "cniVersion":"0.3.1",
+       "cniVersion":"0.4.0",
        "name":"kilo",
        "plugins":[
           {
@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,20 +85,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +110,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -126,7 +131,7 @@ spec:
           readOnly: false
       initContainers:
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         command:
         - /bin/sh
         - -c
@@ -160,7 +165,7 @@ spec:
           path: /opt/cni/bin
       - name: cni-conf-dir
         hostPath:
-          path: /etc/kubernetes/cni/net.d
+          path: /etc/cni/net.d
       - name: kilo-dir
         hostPath:
           path: /var/lib/kilo

manifests/kilo-typhoon-flannel.yaml

@@ -23,14 +23,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,20 +51,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -77,6 +79,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:

manifests/kilo-typhoon.yaml

@@ -8,7 +8,7 @@ metadata:
 data:
   cni-conf.json: |
     {
-       "cniVersion":"0.3.1",
+       "cniVersion":"0.4.0",
        "name":"kilo",
        "plugins":[
           {
@@ -57,14 +57,13 @@ rules:
   - peers
   verbs:
   - list
-  - update
   - watch
 - apiGroups:
   - apiextensions.k8s.io
   resources:
   - customresourcedefinitions
   verbs:
-  - create
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -86,20 +85,23 @@ metadata:
   namespace: kube-system
   labels:
     app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
   template:
     metadata:
       labels:
         app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
     spec:
       serviceAccountName: kilo
       hostNetwork: true
       containers:
       - name: kilo
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         args:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --hostname=$(NODE_NAME)
@@ -108,6 +110,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
         securityContext:
           privileged: true
         volumeMounts:
@@ -126,7 +131,7 @@ spec:
           readOnly: false
       initContainers:
       - name: install-cni
-        image: squat/kilo
+        image: squat/kilo:0.5.0
         command:
         - /bin/sh
         - -c

manifests/peer-validation.yaml

@@ -0,0 +1,173 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kilo
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: "peers.kilo.squat.ai"
+webhooks:
+- name: "peers.kilo.squat.ai"
+  rules:
+  - apiGroups:   ["kilo.squat.ai"]
+    apiVersions: ["v1alpha1"]
+    operations:  ["CREATE","UPDATE"]
+    resources:   ["peers"]
+    scope:       "Cluster"
+  clientConfig:
+    service:
+      namespace: "kilo"
+      name: "peer-validation"
+      path: "/validate"
+  admissionReviewVersions: ["v1"]
+  sideEffects: None
+  timeoutSeconds: 5
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: peer-validation-server
+  namespace: kilo
+  labels:
+    app.kubernetes.io/name: peer-validation-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: peer-validation-server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: peer-validation-server
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+      containers:
+      - name: server
+        image: squat/kilo:0.5.0
+        args:
+        - webhook
+        - --cert-file=/run/secrets/tls/tls.crt
+        - --key-file=/run/secrets/tls/tls.key
+        - --listen-metrics=:1107
+        - --listen=:8443
+        ports:
+        - containerPort: 8443
+          name: webhook
+        - containerPort: 1107
+          name: metrics
+        volumeMounts:
+        - name: tls
+          mountPath: /run/secrets/tls
+          readOnly: true
+      volumes:
+      - name: tls
+        secret:
+          secretName: peer-validation-webhook-tls
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: peer-validation
+  namespace: kilo
+spec:
+  selector:
+    app.kubernetes.io/name: peer-validation-server
+  ports:
+    - port: 443
+      targetPort: webhook
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo-peer-validation
+  namespace: kilo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo-peer-validation
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  resourceNames:
+  - peers.kilo.squat.ai
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo-peer-validation
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo-peer-validation
+subjects:
+  - kind: ServiceAccount
+    namespace: kilo
+    name: kilo-peer-validation
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kilo-peer-validation
+  namespace: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kilo-peer-validation
+  namespace: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kilo-peer-validation
+subjects:
+  - kind: ServiceAccount
+    namespace: kilo
+    name: kilo-peer-validation
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cert-gen
+  namespace: kilo
+spec:
+  template:
+    spec:
+      serviceAccountName: kilo-peer-validation
+      initContainers:
+      - name: create
+        image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0
+        args:
+        - create
+        - --namespace=kilo
+        - --secret-name=peer-validation-webhook-tls
+        - --host=peer-validation,peer-validation.kilo.svc
+        - --key-name=tls.key
+        - --cert-name=tls.crt
+      containers:
+      - name: patch
+        image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0
+        args:
+        - patch
+        - --webhook-name=peers.kilo.squat.ai
+        - --secret-name=peer-validation-webhook-tls
+        - --namespace=kilo
+        - --patch-mutating=false
+      restartPolicy: OnFailure
+  backoffLimit: 4

manifests/podmonitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+  name: kilo
+  namespace: kilo
+spec:
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  podMetricsEndpoints:
+  - interval: 15s
+    port: metrics
+    path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: kilo

manifests/wg-exporter-role-kube-prometheus.yaml

@@ -0,0 +1,56 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: prometheus
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 2.26.0
+  name: prometheus-k8s
+  namespace: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: prometheus
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 2.26.0
+  name: prometheus-k8s
+  namespace: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: monitoring

manifests/wg-exporter.yaml

@@ -0,0 +1,67 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: wg-exporter
+    app.kubernetes.io/part-of: kilo
+  name: wg-exporter
+  namespace: kilo
+spec:
+  namespaceSelector:
+    matchNames:
+    - kilo
+  podMetricsEndpoints:
+  - interval: 15s
+    port: metrics
+    path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: kilo
+      app.kubernetes.io/name: wg-exporter
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/name: wg-exporter
+    app.kubernetes.io/part-of: kilo
+  name: wg-exporter
+  namespace: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: wg-exporter
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: wg-exporter
+        app.kubernetes.io/part-of: kilo
+    spec:
+      containers:
+      - args:
+        - -a
+        - -i=kilo0
+        - -p=9586
+        image: mindflavor/prometheus-wireguard-exporter
+        name: wg-exporter
+        ports:
+        - containerPort: 9586
+          name: metrics
+          protocol: TCP
+        securityContext:
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+      volumes:
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists

pkg/encapsulation/cilium.go

@@ -0,0 +1,111 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encapsulation
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/vishvananda/netlink"
+
+	"github.com/squat/kilo/pkg/iptables"
+)
+
+const ciliumDeviceName = "cilium_host"
+
+type cilium struct {
+	iface    int
+	strategy Strategy
+	ch       chan netlink.LinkUpdate
+	done     chan struct{}
+	// mu guards updates to the iface field.
+	mu sync.Mutex
+}
+
+// NewCilium returns an encapsulator that uses Cilium.
+func NewCilium(strategy Strategy) Encapsulator {
+	return &cilium{
+		ch:       make(chan netlink.LinkUpdate),
+		done:     make(chan struct{}),
+		strategy: strategy,
+	}
+}
+
+// CleanUp close done channel
+func (f *cilium) CleanUp() error {
+	close(f.done)
+	return nil
+}
+
+// Gw returns the correct gateway IP associated with the given node.
+func (f *cilium) Gw(_, _ net.IP, subnet *net.IPNet) net.IP {
+	return subnet.IP
+}
+
+// Index returns the index of the Cilium interface.
+func (f *cilium) Index() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.iface
+}
+
+// Init finds the Cilium interface index.
+func (f *cilium) Init(_ int) error {
+	if err := netlink.LinkSubscribe(f.ch, f.done); err != nil {
+		return fmt.Errorf("failed to subscribe to updates to %s: %v", ciliumDeviceName, err)
+	}
+	go func() {
+		var lu netlink.LinkUpdate
+		for {
+			select {
+			case lu = <-f.ch:
+				if lu.Attrs().Name == ciliumDeviceName {
+					f.mu.Lock()
+					f.iface = lu.Attrs().Index
+					f.mu.Unlock()
+				}
+			case <-f.done:
+				return
+			}
+		}
+	}()
+	i, err := netlink.LinkByName(ciliumDeviceName)
+	if _, ok := err.(netlink.LinkNotFoundError); ok {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to query for Cilium interface: %v", err)
+	}
+	f.mu.Lock()
+	f.iface = i.Attrs().Index
+	f.mu.Unlock()
+	return nil
+}
+
+// Rules is a no-op.
+func (f *cilium) Rules(_ []*net.IPNet) []iptables.Rule {
+	return nil
+}
+
+// Set is a no-op.
+func (f *cilium) Set(_ *net.IPNet) error {
+	return nil
+}
+
+// Strategy returns the configured strategy for encapsulation.
+func (f *cilium) Strategy() Strategy {
+	return f.strategy
+}

pkg/encapsulation/flannel.go

@@ -56,6 +56,8 @@ func (f *flannel) Gw(_, _ net.IP, subnet *net.IPNet) net.IP {
 
 // Index returns the index of the Flannel interface.
 func (f *flannel) Index() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
 	return f.iface
 }
 

pkg/encapsulation/ipip.go

@@ -67,17 +67,18 @@ func (i *ipip) Init(base int) error {
 // when traffic between nodes must be encapsulated.
 func (i *ipip) Rules(nodes []*net.IPNet) []iptables.Rule {
 	var rules []iptables.Rule
+	proto := ipipProtocolName()
 	rules = append(rules, iptables.NewIPv4Chain("filter", "KILO-IPIP"))
 	rules = append(rules, iptables.NewIPv6Chain("filter", "KILO-IPIP"))
-	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-p", "4", "-j", "KILO-IPIP"))
+	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-j", "KILO-IPIP"))
-	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-p", "4", "-j", "KILO-IPIP"))
+	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: jump to IPIP chain", "-j", "KILO-IPIP"))
 	for _, n := range nodes {
 		// Accept encapsulated traffic from peers.
-		rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(n.IP)), "filter", "KILO-IPIP", "-m", "comment", "--comment", "Kilo: allow IPIP traffic", "-s", n.IP.String(), "-j", "ACCEPT"))
+		rules = append(rules, iptables.NewRule(iptables.GetProtocol(n.IP), "filter", "KILO-IPIP", "-s", n.String(), "-m", "comment", "--comment", "Kilo: allow IPIP traffic", "-j", "ACCEPT"))
 	}
 	// Drop all other IPIP traffic.
-	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-p", "4", "-j", "DROP"))
+	rules = append(rules, iptables.NewIPv4Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-j", "DROP"))
-	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-p", "4", "-j", "DROP"))
+	rules = append(rules, iptables.NewIPv6Rule("filter", "INPUT", "-p", proto, "-m", "comment", "--comment", "Kilo: reject other IPIP traffic", "-j", "DROP"))
 
 	return rules
 }

pkg/encapsulation/ipip_cgo.go

@@ -1,4 +1,4 @@
-// Copyright 2015 CNI authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,16 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package ip
+//go:build cgo
+// +build cgo
 
-import (
+package encapsulation
-	"net"
 
-	"github.com/vishvananda/netlink"
+/*
-)
+#include <netdb.h>
+*/
+import "C"
 
-// AddDefaultRoute sets the default route on the given gateway.
+func ipipProtocolName() string {
-func AddDefaultRoute(gw net.IP, dev netlink.Link) error {
+	return C.GoString(C.getprotobynumber(4).p_name)
-	_, defNet, _ := net.ParseCIDR("0.0.0.0/0")
-	return AddRoute(defNet, gw, dev)
 }

pkg/encapsulation/ipip_nocgo.go

@@ -0,0 +1,25 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !cgo
+// +build !cgo
+
+package encapsulation
+
+// If we can determine the protocol name at runtime
+// by looking it up in the protocols database, assume `ipencap`
+// as this is the value in Kilo's container image.
+func ipipProtocolName() string {
+	return "ipencap"
+}

pkg/encapsulation/noop.go

@@ -0,0 +1,59 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encapsulation
+
+import (
+	"net"
+
+	"github.com/squat/kilo/pkg/iptables"
+)
+
+// Noop is an encapsulation that does nothing.
+type Noop Strategy
+
+// CleanUp will also do nothing.
+func (n Noop) CleanUp() error {
+	return nil
+}
+
+// Gw will also do nothing.
+func (n Noop) Gw(_ net.IP, _ net.IP, _ *net.IPNet) net.IP {
+	return nil
+}
+
+// Index will also do nothing.
+func (n Noop) Index() int {
+	return 0
+}
+
+// Init will also do nothing.
+func (n Noop) Init(_ int) error {
+	return nil
+}
+
+// Rules will also do nothing.
+func (n Noop) Rules(_ []*net.IPNet) []iptables.Rule {
+	return nil
+}
+
+// Set will also do nothing.
+func (n Noop) Set(_ *net.IPNet) error {
+	return nil
+}
+
+// Strategy will finally do nothing.
+func (n Noop) Strategy() Strategy {
+	return Strategy(n)
+}

pkg/iptables/fake.go

@@ -16,6 +16,8 @@ package iptables
 
 import (
 	"fmt"
+	"strings"
+	"sync/atomic"
 
 	"github.com/coreos/go-iptables/iptables"
 )
@@ -38,12 +40,14 @@ func (s statusError) ExitStatus() int {
 }
 
 type fakeClient struct {
+	calls   uint64
 	storage []Rule
 }
 
 var _ Client = &fakeClient{}
 
 func (f *fakeClient) AppendUnique(table, chain string, spec ...string) error {
+	atomic.AddUint64(&f.calls, 1)
 	exists, err := f.Exists(table, chain, spec...)
 	if err != nil {
 		return err
@@ -56,6 +60,7 @@ func (f *fakeClient) AppendUnique(table, chain string, spec ...string) error {
 }
 
 func (f *fakeClient) Delete(table, chain string, spec ...string) error {
+	atomic.AddUint64(&f.calls, 1)
 	r := &rule{table: table, chain: chain, spec: spec}
 	for i := range f.storage {
 		if f.storage[i].String() == r.String() {
@@ -69,6 +74,7 @@ func (f *fakeClient) Delete(table, chain string, spec ...string) error {
 }
 
 func (f *fakeClient) Exists(table, chain string, spec ...string) (bool, error) {
+	atomic.AddUint64(&f.calls, 1)
 	r := &rule{table: table, chain: chain, spec: spec}
 	for i := range f.storage {
 		if f.storage[i].String() == r.String() {
@@ -78,7 +84,22 @@ func (f *fakeClient) Exists(table, chain string, spec ...string) (bool, error) {
 	return false, nil
 }
 
+func (f *fakeClient) List(table, chain string) ([]string, error) {
+	atomic.AddUint64(&f.calls, 1)
+	var rs []string
+	for i := range f.storage {
+		switch r := f.storage[i].(type) {
+		case *rule:
+			if r.table == table && r.chain == chain {
+				rs = append(rs, strings.TrimSpace(strings.TrimPrefix(r.String(), table)))
+			}
+		}
+	}
+	return rs, nil
+}
+
 func (f *fakeClient) ClearChain(table, name string) error {
+	atomic.AddUint64(&f.calls, 1)
 	for i := range f.storage {
 		r, ok := f.storage[i].(*rule)
 		if !ok {
@@ -90,10 +111,14 @@ func (f *fakeClient) ClearChain(table, name string) error {
 			}
 		}
 	}
-	return f.DeleteChain(table, name)
+	if err := f.DeleteChain(table, name); err != nil {
+		return err
+	}
+	return f.NewChain(table, name)
 }
 
 func (f *fakeClient) DeleteChain(table, name string) error {
+	atomic.AddUint64(&f.calls, 1)
 	for i := range f.storage {
 		r, ok := f.storage[i].(*rule)
 		if !ok {
@@ -116,6 +141,7 @@ func (f *fakeClient) DeleteChain(table, name string) error {
 }
 
 func (f *fakeClient) NewChain(table, name string) error {
+	atomic.AddUint64(&f.calls, 1)
 	c := &chain{table: table, chain: name}
 	for i := range f.storage {
 		if f.storage[i].String() == c.String() {
@@ -125,3 +151,17 @@ func (f *fakeClient) NewChain(table, name string) error {
 	f.storage = append(f.storage, c)
 	return nil
 }
+
+func (f *fakeClient) ListChains(table string) ([]string, error) {
+	atomic.AddUint64(&f.calls, 1)
+	var cs []string
+	for i := range f.storage {
+		switch c := f.storage[i].(type) {
+		case *chain:
+			if c.table == table {
+				cs = append(cs, c.chain)
+			}
+		}
+	}
+	return cs, nil
+}

pkg/iptables/iptables.go

@@ -16,14 +16,32 @@ package iptables
 
 import (
 	"fmt"
+	"io"
 	"net"
-	"strings"
+	"os"
 	"sync"
 	"time"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
 )
 
+const ipv6ModuleDisabledPath = "/sys/module/ipv6/parameters/disable"
+
+func ipv6Disabled() (bool, error) {
+	f, err := os.Open(ipv6ModuleDisabledPath)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+	disabled := make([]byte, 1)
+	if _, err = io.ReadFull(f, disabled); err != nil {
+		return false, err
+	}
+	return disabled[0] == '1', nil
+}
+
 // Protocol represents an IP protocol.
 type Protocol byte
 
@@ -35,11 +53,11 @@ const (
 )
 
 // GetProtocol will return a protocol from the length of an IP address.
-func GetProtocol(length int) Protocol {
+func GetProtocol(ip net.IP) Protocol {
-	if length == net.IPv6len {
+	if len(ip) == net.IPv4len || ip.To4() != nil {
-		return ProtocolIPv6
-	}
 		return ProtocolIPv4
+	}
+	return ProtocolIPv6
 }
 
 // Client represents any type that can administer iptables rules.
@@ -47,9 +65,11 @@ type Client interface {
 	AppendUnique(table string, chain string, rule ...string) error
 	Delete(table string, chain string, rule ...string) error
 	Exists(table string, chain string, rule ...string) (bool, error)
+	List(table string, chain string) ([]string, error)
 	ClearChain(table string, chain string) error
 	DeleteChain(table string, chain string) error
 	NewChain(table string, chain string) error
+	ListChains(table string) ([]string, error)
 }
 
 // Rule is an interface for interacting with iptables objects.
@@ -107,7 +127,17 @@ func (r *rule) String() string {
 	if r == nil {
 		return ""
 	}
-	return fmt.Sprintf("%s_%s_%s", r.table, r.chain, strings.Join(r.spec, "_"))
+	spec := r.table + " -A " + r.chain
+	for i, s := range r.spec {
+		spec += " "
+		// If this is the content of a comment, wrap the value in quotes.
+		if i > 0 && r.spec[i-1] == "--comment" {
+			spec += `"` + s + `"`
+		} else {
+			spec += s
+		}
+	}
+	return spec
 }
 
 func (r *rule) Proto() Protocol {
@@ -132,6 +162,7 @@ func NewIPv6Chain(table, name string) Rule {
 }
 
 func (c *chain) Add(client Client) error {
+	// Note: `ClearChain` creates a chain if it does not exist.
 	if err := client.ClearChain(c.table, c.chain); err != nil {
 		return fmt.Errorf("failed to add iptables chain: %v", err)
 	}
@@ -171,41 +202,90 @@ func (c *chain) String() string {
 	if c == nil {
 		return ""
 	}
-	return fmt.Sprintf("%s_%s", c.table, c.chain)
+	return chainToString(c.table, c.chain)
 }
 
 func (c *chain) Proto() Protocol {
 	return c.proto
 }
 
+func chainToString(table, chain string) string {
+	return fmt.Sprintf("%s -N %s", table, chain)
+}
+
 // Controller is able to reconcile a given set of iptables rules.
 type Controller struct {
 	v4           Client
 	v6           Client
 	errors       chan error
+	logger       log.Logger
+	resyncPeriod time.Duration
 
 	sync.Mutex
 	rules      []Rule
 	subscribed bool
 }
 
+// ControllerOption modifies the controller's configuration.
+type ControllerOption func(h *Controller)
+
+// WithLogger adds a logger to the controller.
+func WithLogger(logger log.Logger) ControllerOption {
+	return func(c *Controller) {
+		c.logger = logger
+	}
+}
+
+// WithResyncPeriod modifies how often the controller reconciles.
+func WithResyncPeriod(resyncPeriod time.Duration) ControllerOption {
+	return func(c *Controller) {
+		c.resyncPeriod = resyncPeriod
+	}
+}
+
+// WithClients adds iptables clients to the controller.
+func WithClients(v4, v6 Client) ControllerOption {
+	return func(c *Controller) {
+		c.v4 = v4
+		c.v6 = v6
+	}
+}
+
 // New generates a new iptables rules controller.
-// It expects an IP address length to determine
+// If no options are given, IPv4 and IPv6 clients
-// whether to operate in IPv4 or IPv6 mode.
+// will be instantiated using the regular iptables backend.
-func New() (*Controller, error) {
+func New(opts ...ControllerOption) (*Controller, error) {
+	c := &Controller{
+		errors: make(chan error),
+		logger: log.NewNopLogger(),
+	}
+	for _, o := range opts {
+		o(c)
+	}
+	if c.v4 == nil {
 		v4, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create iptables IPv4 client: %v", err)
 		}
+		c.v4 = v4
+	}
+	if c.v6 == nil {
+		disabled, err := ipv6Disabled()
+		if err != nil {
+			return nil, fmt.Errorf("failed to check IPv6 status: %v", err)
+		}
+		if disabled {
+			level.Info(c.logger).Log("msg", "IPv6 is disabled in the kernel; disabling the IPv6 iptables controller")
+			c.v6 = &fakeClient{}
+		} else {
 			v6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
 			if err != nil {
 				return nil, fmt.Errorf("failed to create iptables IPv6 client: %v", err)
 			}
-	return &Controller{
+			c.v6 = v6
-		v4:     v4,
+		}
-		v6:     v6,
+	}
-		errors: make(chan error),
+	return c, nil
-	}, nil
 }
 
 // Run watches for changes to iptables rules and reconciles
@@ -220,16 +300,18 @@ func (c *Controller) Run(stop <-chan struct{}) (<-chan error, error) {
 	c.subscribed = true
 	c.Unlock()
 	go func() {
+		t := time.NewTimer(c.resyncPeriod)
 		defer close(c.errors)
 		for {
 			select {
-			case <-time.After(5 * time.Second):
+			case <-t.C:
-			case <-stop:
-				return
-			}
 				if err := c.reconcile(); err != nil {
 					nonBlockingSend(c.errors, fmt.Errorf("failed to reconcile rules: %v", err))
 				}
+				t.Reset(c.resyncPeriod)
+			case <-stop:
+				return
+			}
 		}
 	}()
 	return c.errors, nil
@@ -242,12 +324,14 @@ func (c *Controller) Run(stop <-chan struct{}) (<-chan error, error) {
 func (c *Controller) reconcile() error {
 	c.Lock()
 	defer c.Unlock()
+	var rc ruleCache
 	for i, r := range c.rules {
-		ok, err := r.Exists(c.client(r.Proto()))
+		ok, err := rc.exists(c.client(r.Proto()), r)
 		if err != nil {
 			return fmt.Errorf("failed to check if rule exists: %v", err)
 		}
 		if !ok {
+			level.Info(c.logger).Log("msg", fmt.Sprintf("applying %d iptables rules", len(c.rules)-i))
 			if err := c.resetFromIndex(i, c.rules); err != nil {
 				return fmt.Errorf("failed to add rule: %v", err)
 			}

pkg/iptables/iptables_test.go

@@ -83,10 +83,11 @@ func TestSet(t *testing.T) {
 			},
 		},
 	} {
-		controller := &Controller{}
 		client := &fakeClient{}
-		controller.v4 = client
+		controller, err := New(WithClients(client, client))
-		controller.v6 = client
+		if err != nil {
+			t.Fatalf("test case %q: got unexpected error instantiating controller: %v", tc.name, err)
+		}
 		for i := range tc.sets {
 			if err := controller.Set(tc.sets[i]); err != nil {
 				t.Fatalf("test case %q: got unexpected error seting rule set %d: %v", tc.name, i, err)
@@ -139,10 +140,11 @@ func TestCleanUp(t *testing.T) {
 			rules: []Rule{rules[0], rules[1]},
 		},
 	} {
-		controller := &Controller{}
 		client := &fakeClient{}
-		controller.v4 = client
+		controller, err := New(WithClients(client, client))
-		controller.v6 = client
+		if err != nil {
+			t.Fatalf("test case %q: got unexpected error instantiating controller: %v", tc.name, err)
+		}
 		if err := controller.Set(tc.rules); err != nil {
 			t.Fatalf("test case %q: Set should not fail: %v", tc.name, err)
 		}

pkg/iptables/rulecache.go

@@ -0,0 +1,106 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ruleCacheFlag byte
+
+const (
+	exists ruleCacheFlag = 1 << iota
+	populated
+)
+
+type isNotExistError interface {
+	error
+	IsNotExist() bool
+}
+
+// ruleCache is a lazy cache that can be used to
+// check if a given rule or chain exists in an iptables
+// table.
+type ruleCache [2]map[string]ruleCacheFlag
+
+func (rc *ruleCache) populateTable(c Client, proto Protocol, table string) error {
+	// If the table already exists in the destination map,
+	// exit early since it has already been populated.
+	if rc[proto][table]&populated != 0 {
+		return nil
+	}
+	cs, err := c.ListChains(table)
+	if err != nil {
+		return fmt.Errorf("failed to populate chains for table %q: %v", table, err)
+	}
+	rc[proto][table] = exists | populated
+	for i := range cs {
+		rc[proto][chainToString(table, cs[i])] |= exists
+	}
+	return nil
+}
+
+func (rc *ruleCache) populateChain(c Client, proto Protocol, table, chain string) error {
+	// If the destination chain true, then it has already been populated.
+	if rc[proto][chainToString(table, chain)]&populated != 0 {
+		return nil
+	}
+	rs, err := c.List(table, chain)
+	if err != nil {
+		if existsErr, ok := err.(isNotExistError); ok && existsErr.IsNotExist() {
+			rc[proto][chainToString(table, chain)] = populated
+			return nil
+		}
+		return fmt.Errorf("failed to populate rules in chain %q for table %q: %v", chain, table, err)
+	}
+	for i := range rs {
+		rc[proto][strings.Join([]string{table, rs[i]}, " ")] = exists
+	}
+	// If there are rules on the chain, then the chain exists too.
+	if len(rs) > 0 {
+		rc[proto][chainToString(table, chain)] = exists
+	}
+	rc[proto][chainToString(table, chain)] |= populated
+	return nil
+}
+
+func (rc *ruleCache) populateRules(c Client, r Rule) error {
+	// Ensure a map for the proto exists.
+	if rc[r.Proto()] == nil {
+		rc[r.Proto()] = make(map[string]ruleCacheFlag)
+	}
+
+	if ch, ok := r.(*chain); ok {
+		return rc.populateTable(c, r.Proto(), ch.table)
+	}
+
+	ru := r.(*rule)
+	return rc.populateChain(c, r.Proto(), ru.table, ru.chain)
+}
+
+func (rc *ruleCache) exists(c Client, r Rule) (bool, error) {
+	// Exit early if the exact rule exists by name.
+	if rc[r.Proto()][r.String()]&exists != 0 {
+		return true, nil
+	}
+
+	// Otherwise, populate the respective rules.
+	if err := rc.populateRules(c, r); err != nil {
+		return false, err
+	}
+
+	return rc[r.Proto()][r.String()]&exists != 0, nil
+}

pkg/iptables/rulecache_test.go

@@ -0,0 +1,125 @@
+// Copyright 2021 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"testing"
+)
+
+func TestRuleCache(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		rules []Rule
+		check []Rule
+		out   []bool
+		calls uint64
+	}{
+		{
+			name:  "empty",
+			rules: nil,
+			check: []Rule{rules[0]},
+			out:   []bool{false},
+			calls: 1,
+		},
+		{
+			name:  "single negative",
+			rules: []Rule{rules[1]},
+			check: []Rule{rules[0]},
+			out:   []bool{false},
+			calls: 1,
+		},
+		{
+			name:  "single positive",
+			rules: []Rule{rules[1]},
+			check: []Rule{rules[1]},
+			out:   []bool{true},
+			calls: 1,
+		},
+		{
+			name:  "single chain",
+			rules: []Rule{&chain{"nat", "KILO-NAT", ProtocolIPv4}},
+			check: []Rule{&chain{"nat", "KILO-NAT", ProtocolIPv4}},
+			out:   []bool{true},
+			calls: 1,
+		},
+		{
+			name:  "rule on chain means chain exists",
+			rules: []Rule{rules[0]},
+			check: []Rule{rules[0], &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{true, true},
+			calls: 1,
+		},
+		{
+			name:  "rule on chain does not mean table is fully populated",
+			rules: []Rule{rules[0], &chain{"filter", "INPUT", ProtocolIPv4}},
+			check: []Rule{rules[0], &chain{"filter", "OUTPUT", ProtocolIPv4}, &chain{"filter", "INPUT", ProtocolIPv4}},
+			out:   []bool{true, false, true},
+			calls: 2,
+		},
+		{
+			name:  "multiple rules on chain",
+			rules: []Rule{rules[0], rules[1]},
+			check: []Rule{rules[0], rules[1], &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{true, true, true},
+			calls: 1,
+		},
+		{
+			name:  "checking rule on chain does not mean chain exists",
+			rules: nil,
+			check: []Rule{rules[0], &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{false, false},
+			calls: 2,
+		},
+		{
+			name:  "multiple chains on same table",
+			rules: nil,
+			check: []Rule{&chain{"filter", "INPUT", ProtocolIPv4}, &chain{"filter", "FORWARD", ProtocolIPv4}},
+			out:   []bool{false, false},
+			calls: 1,
+		},
+		{
+			name:  "multiple chains on different table",
+			rules: nil,
+			check: []Rule{&chain{"filter", "INPUT", ProtocolIPv4}, &chain{"nat", "POSTROUTING", ProtocolIPv4}},
+			out:   []bool{false, false},
+			calls: 2,
+		},
+	} {
+		controller := &Controller{}
+		client := &fakeClient{}
+		controller.v4 = client
+		controller.v6 = client
+		if err := controller.Set(tc.rules); err != nil {
+			t.Fatalf("test case %q: Set should not fail: %v", tc.name, err)
+		}
+		// Reset the client's calls so we can examine how many times
+		// the rule cache performs operations.
+		client.calls = 0
+		var rc ruleCache
+		for i := range tc.check {
+			ok, err := rc.exists(controller.client(tc.check[i].Proto()), tc.check[i])
+			if err != nil {
+				t.Fatalf("test case %q check %d: check should not fail: %v", tc.name, i, err)
+			}
+			if ok != tc.out[i] {
+				t.Errorf("test case %q check %d: expected %t, got %t", tc.name, i, tc.out[i], ok)
+			}
+		}
+		if client.calls != tc.calls {
+			t.Errorf("test case %q: expected client to be called %d times, got %d", tc.name, tc.calls, client.calls)
+		}
+	}
+
+}

pkg/k8s/apis/kilo/register.go

@@ -16,5 +16,5 @@ package kilo
 
 const (
 	// GroupName contains the API group name for Kilo API group.
-	GroupName = "kilo"
+	GroupName = "kilo.squat.ai"
 )

pkg/k8s/apis/kilo/v1alpha1/doc.go

@@ -15,5 +15,5 @@
 // +k8s:deepcopy-gen=package,register
 
 // Package v1alpha1 is the v1alpha1 version of the API.
-// +groupName=kilo
+// +groupName=kilo.squat.ai
 package v1alpha1

pkg/k8s/apis/kilo/v1alpha1/types.go

@@ -15,13 +15,16 @@
 package v1alpha1
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/validation"
 )
 
 const (
@@ -45,6 +48,7 @@ var PeerShortNames = []string{"peer"}
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +k8s:openapi-gen=true
+// +kubebuilder:resource:scope=Cluster
 
 // Peer is a WireGuard peer that should have access to the VPN.
 type Peer struct {
@@ -74,19 +78,30 @@ type PeerSpec struct {
 	PersistentKeepalive int `json:"persistentKeepalive,omitempty"`
 	// PresharedKey is the optional symmetric encryption key for the peer.
 	// +optional
-	PresharedKey string `json:"presharedKey"`
+	PresharedKey string `json:"presharedKey,omitempty"`
 	// PublicKey is the WireGuard public key for the peer.
 	PublicKey string `json:"publicKey"`
 }
 
-// PeerEndpoint represents a WireGuard enpoint, which is a ip:port tuple.
+// PeerEndpoint represents a WireGuard endpoint, which is an IP:port tuple.
 type PeerEndpoint struct {
-	// IP must be a valid IP address.
+	// DNSOrIP is a DNS name or an IP address.
-	IP string `json:"ip"`
+	DNSOrIP `json:"dnsOrIP"`
 	// Port must be a valid port number.
 	Port uint32 `json:"port"`
 }
 
+// DNSOrIP represents either a DNS name or an IP address.
+// When both are given, the IP address, as it is more specific, override the DNS name.
+type DNSOrIP struct {
+	// DNS must be a valid RFC 1123 subdomain.
+	// +optional
+	DNS string `json:"dns,omitempty"`
+	// IP must be a valid IP address.
+	// +optional
+	IP string `json:"ip,omitempty"`
+}
+
 // PeerName is the peer resource's FQDN.
 var PeerName = PeerPlural + "." + GroupName
 
@@ -121,24 +136,35 @@ func (p *Peer) Copy() *Peer {
 func (p *Peer) Validate() error {
 	for _, ip := range p.Spec.AllowedIPs {
 		if _, n, err := net.ParseCIDR(ip); err != nil {
-			return fmt.Errorf("failed to parse %q as a valid IP address: %v", ip, err)
+			return fmt.Errorf("failed to parse %q as a valid IP address: %w", ip, err)
 		} else if n == nil {
 			return fmt.Errorf("got invalid IP address for %q", ip)
 		}
 	}
 	if p.Spec.Endpoint != nil {
-		if net.ParseIP(p.Spec.Endpoint.IP) == nil {
+		if p.Spec.Endpoint.IP == "" && p.Spec.Endpoint.DNS == "" {
+			return errors.New("either an endpoint DNS name IP address must be given")
+		}
+		if p.Spec.Endpoint.DNS != "" {
+			if errs := validation.IsDNS1123Subdomain(p.Spec.Endpoint.DNS); len(errs) != 0 {
+				return errors.New(strings.Join(errs, "; "))
+			}
+		}
+		if p.Spec.Endpoint.IP != "" && net.ParseIP(p.Spec.Endpoint.IP) == nil {
 			return fmt.Errorf("failed to parse %q as a valid IP address", p.Spec.Endpoint.IP)
 		}
-		if p.Spec.Endpoint.Port == 0 {
+		if 1 > p.Spec.Endpoint.Port || p.Spec.Endpoint.Port > 65535 {
 			return fmt.Errorf("port must be a valid UDP port number, got %d", p.Spec.Endpoint.Port)
 		}
 	}
 	if p.Spec.PersistentKeepalive < 0 {
 		return fmt.Errorf("persistent keepalive must be greater than or equal to zero; got %q", p.Spec.PersistentKeepalive)
 	}
-	if len(p.Spec.PublicKey) == 0 {
+	if b, err := base64.StdEncoding.DecodeString(p.Spec.PublicKey); err != nil {
-		return errors.New("public keys cannot be empty")
+		return fmt.Errorf("WireGuard public key is not base64 encoded: %w", err)
+		// Since WireGuard is using Curve25519 for the key exchange, the key length of 256 bits should not change in the near future.
+	} else if len(b) != 32 {
+		return errors.New("WireGuard public key has invalid length")
 	}
 	return nil
 }
@@ -148,6 +174,9 @@ func (p *Peer) Validate() error {
 // PeerList is a list of peers.
 type PeerList struct {
 	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	// +optional
 	metav1.ListMeta `json:"metadata,omitempty"`
 	// List of peers.
 	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md

pkg/k8s/apis/kilo/v1alpha1/zz_generated.deepcopy.go

@@ -1,6 +1,7 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,6 +23,22 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DNSOrIP) DeepCopyInto(out *DNSOrIP) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSOrIP.
+func (in *DNSOrIP) DeepCopy() *DNSOrIP {
+	if in == nil {
+		return nil
+	}
+	out := new(DNSOrIP)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Peer) DeepCopyInto(out *Peer) {
 	*out = *in
@@ -52,6 +69,7 @@ func (in *Peer) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PeerEndpoint) DeepCopyInto(out *PeerEndpoint) {
 	*out = *in
+	out.DNSOrIP = in.DNSOrIP
 	return
 }
 
@@ -69,7 +87,7 @@ func (in *PeerEndpoint) DeepCopy() *PeerEndpoint {
 func (in *PeerList) DeepCopyInto(out *PeerList) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
-	out.ListMeta = in.ListMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
 	if in.Items != nil {
 		in, out := &in.Items, &out.Items
 		*out = make([]Peer, len(*in))

pkg/k8s/backend.go

@@ -15,6 +15,7 @@
 package k8s
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -24,15 +25,15 @@ import (
 	"strings"
 	"time"
 
-	crdutils "github.com/ant31/crd-validation/pkg"
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
-	"k8s.io/apimachinery/pkg/util/validation"
 	v1informers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
 	v1listers "k8s.io/client-go/listers/core/v1"
@@ -59,12 +60,17 @@ const (
 	locationAnnotationKey        = "kilo.squat.ai/location"
 	persistentKeepaliveKey       = "kilo.squat.ai/persistent-keepalive"
 	wireGuardIPAnnotationKey     = "kilo.squat.ai/wireguard-ip"
-
+	discoveredEndpointsKey       = "kilo.squat.ai/discovered-endpoints"
-	regionLabelKey  = "topology.kubernetes.io/region"
+	allowedLocationIPsKey        = "kilo.squat.ai/allowed-location-ips"
+	granularityKey               = "kilo.squat.ai/granularity"
+	// RegionLabelKey is the key for the well-known Kubernetes topology region label.
+	RegionLabelKey  = "topology.kubernetes.io/region"
 	jsonPatchSlash  = "~1"
 	jsonRemovePatch = `{"op": "remove", "path": "%s"}`
 )
 
+var logger = log.NewNopLogger()
+
 type backend struct {
 	nodes *nodeBackend
 	peers *peerBackend
@@ -85,6 +91,7 @@ type nodeBackend struct {
 	events        chan *mesh.NodeEvent
 	informer      cache.SharedIndexInformer
 	lister        v1listers.NodeLister
+	topologyLabel string
 }
 
 type peerBackend struct {
@@ -96,16 +103,19 @@ type peerBackend struct {
 }
 
 // New creates a new instance of a mesh.Backend.
-func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Interface) mesh.Backend {
+func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Interface, topologyLabel string, l log.Logger) mesh.Backend {
 	ni := v1informers.NewNodeInformer(c, 5*time.Minute, nil)
 	pi := v1alpha1informers.NewPeerInformer(kc, 5*time.Minute, nil)
 
+	logger = l
+
 	return &backend{
 		&nodeBackend{
 			client:        c,
 			events:        make(chan *mesh.NodeEvent),
 			informer:      ni,
 			lister:        v1listers.NewNodeLister(ni.GetIndexer()),
+			topologyLabel: topologyLabel,
 		},
 		&peerBackend{
 			client:           kc,
@@ -118,15 +128,17 @@ func New(c kubernetes.Interface, kc kiloclient.Interface, ec apiextensions.Inter
 }
 
 // CleanUp removes configuration applied to the backend.
-func (nb *nodeBackend) CleanUp(name string) error {
+func (nb *nodeBackend) CleanUp(ctx context.Context, name string) error {
 	patch := []byte("[" + strings.Join([]string{
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(endpointAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(internalIPAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(keyAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(lastSeenAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(wireGuardIPAnnotationKey, "/", jsonPatchSlash, 1))),
+		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(discoveredEndpointsKey, "/", jsonPatchSlash, 1))),
+		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(granularityKey, "/", jsonPatchSlash, 1))),
 	}, ",") + "]")
-	if _, err := nb.client.CoreV1().Nodes().Patch(name, types.JSONPatchType, patch); err != nil {
+	if _, err := nb.client.CoreV1().Nodes().Patch(ctx, name, types.JSONPatchType, patch, metav1.PatchOptions{}); err != nil {
 		return fmt.Errorf("failed to patch node: %v", err)
 	}
 	return nil
@@ -138,14 +150,14 @@ func (nb *nodeBackend) Get(name string) (*mesh.Node, error) {
 	if err != nil {
 		return nil, err
 	}
-	return translateNode(n), nil
+	return translateNode(n, nb.topologyLabel), nil
 }
 
 // Init initializes the backend; for this backend that means
 // syncing the informer cache.
-func (nb *nodeBackend) Init(stop <-chan struct{}) error {
+func (nb *nodeBackend) Init(ctx context.Context) error {
-	go nb.informer.Run(stop)
+	go nb.informer.Run(ctx.Done())
-	if ok := cache.WaitForCacheSync(stop, func() bool {
+	if ok := cache.WaitForCacheSync(ctx.Done(), func() bool {
 		return nb.informer.HasSynced()
 	}); !ok {
 		return errors.New("failed to sync node cache")
@@ -158,7 +170,7 @@ func (nb *nodeBackend) Init(stop <-chan struct{}) error {
 					// Failed to decode Node; ignoring...
 					return
 				}
-				nb.events <- &mesh.NodeEvent{Type: mesh.AddEvent, Node: translateNode(n)}
+				nb.events <- &mesh.NodeEvent{Type: mesh.AddEvent, Node: translateNode(n, nb.topologyLabel)}
 			},
 			UpdateFunc: func(old, obj interface{}) {
 				n, ok := obj.(*v1.Node)
@@ -171,7 +183,7 @@ func (nb *nodeBackend) Init(stop <-chan struct{}) error {
 					// Failed to decode Node; ignoring...
 					return
 				}
-				nb.events <- &mesh.NodeEvent{Type: mesh.UpdateEvent, Node: translateNode(n), Old: translateNode(o)}
+				nb.events <- &mesh.NodeEvent{Type: mesh.UpdateEvent, Node: translateNode(n, nb.topologyLabel), Old: translateNode(o, nb.topologyLabel)}
 			},
 			DeleteFunc: func(obj interface{}) {
 				n, ok := obj.(*v1.Node)
@@ -179,7 +191,7 @@ func (nb *nodeBackend) Init(stop <-chan struct{}) error {
 					// Failed to decode Node; ignoring...
 					return
 				}
-				nb.events <- &mesh.NodeEvent{Type: mesh.DeleteEvent, Node: translateNode(n)}
+				nb.events <- &mesh.NodeEvent{Type: mesh.DeleteEvent, Node: translateNode(n, nb.topologyLabel)}
 			},
 		},
 	)
@@ -194,27 +206,41 @@ func (nb *nodeBackend) List() ([]*mesh.Node, error) {
 	}
 	nodes := make([]*mesh.Node, len(ns))
 	for i := range ns {
-		nodes[i] = translateNode(ns[i])
+		nodes[i] = translateNode(ns[i], nb.topologyLabel)
 	}
 	return nodes, nil
 }
 
 // Set sets the fields of a node.
-func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
+func (nb *nodeBackend) Set(ctx context.Context, name string, node *mesh.Node) error {
 	old, err := nb.lister.Get(name)
 	if err != nil {
 		return fmt.Errorf("failed to find node: %v", err)
 	}
 	n := old.DeepCopy()
 	n.ObjectMeta.Annotations[endpointAnnotationKey] = node.Endpoint.String()
+	if node.InternalIP == nil {
+		n.ObjectMeta.Annotations[internalIPAnnotationKey] = ""
+	} else {
 		n.ObjectMeta.Annotations[internalIPAnnotationKey] = node.InternalIP.String()
-	n.ObjectMeta.Annotations[keyAnnotationKey] = string(node.Key)
+	}
+	n.ObjectMeta.Annotations[keyAnnotationKey] = node.Key.String()
 	n.ObjectMeta.Annotations[lastSeenAnnotationKey] = strconv.FormatInt(node.LastSeen, 10)
 	if node.WireGuardIP == nil {
 		n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = ""
 	} else {
 		n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = node.WireGuardIP.String()
 	}
+	if node.DiscoveredEndpoints == nil {
+		n.ObjectMeta.Annotations[discoveredEndpointsKey] = ""
+	} else {
+		discoveredEndpoints, err := json.Marshal(node.DiscoveredEndpoints)
+		if err != nil {
+			return err
+		}
+		n.ObjectMeta.Annotations[discoveredEndpointsKey] = string(discoveredEndpoints)
+	}
+	n.ObjectMeta.Annotations[granularityKey] = string(node.Granularity)
 	oldData, err := json.Marshal(old)
 	if err != nil {
 		return err
@@ -227,7 +253,7 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
 	if err != nil {
 		return fmt.Errorf("failed to create patch for node %q: %v", n.Name, err)
 	}
-	if _, err = nb.client.CoreV1().Nodes().Patch(name, types.StrategicMergePatchType, patch); err != nil {
+	if _, err = nb.client.CoreV1().Nodes().Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
 		return fmt.Errorf("failed to patch node: %v", err)
 	}
 	return nil
@@ -239,7 +265,7 @@ func (nb *nodeBackend) Watch() <-chan *mesh.NodeEvent {
 }
 
 // translateNode translates a Kubernetes Node to a mesh.Node.
-func translateNode(node *v1.Node) *mesh.Node {
+func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 	if node == nil {
 		return nil
 	}
@@ -253,26 +279,30 @@ func translateNode(node *v1.Node) *mesh.Node {
 	// Allow the region to be overridden by an explicit location.
 	location, ok := node.ObjectMeta.Annotations[locationAnnotationKey]
 	if !ok {
-		location = node.ObjectMeta.Labels[regionLabelKey]
+		location = node.ObjectMeta.Labels[topologyLabel]
 	}
 	// Allow the endpoint to be overridden.
-	endpoint := parseEndpoint(node.ObjectMeta.Annotations[forceEndpointAnnotationKey])
+	endpoint := wireguard.ParseEndpoint(node.ObjectMeta.Annotations[forceEndpointAnnotationKey])
 	if endpoint == nil {
-		endpoint = parseEndpoint(node.ObjectMeta.Annotations[endpointAnnotationKey])
+		endpoint = wireguard.ParseEndpoint(node.ObjectMeta.Annotations[endpointAnnotationKey])
 	}
 	// Allow the internal IP to be overridden.
 	internalIP := normalizeIP(node.ObjectMeta.Annotations[forceInternalIPAnnotationKey])
 	if internalIP == nil {
 		internalIP = normalizeIP(node.ObjectMeta.Annotations[internalIPAnnotationKey])
 	}
-	// Set Wireguard PersistentKeepalive setting for the node.
+	// Set the ForceInternalIP flag, if force-internal-ip annotation was set to "".
-	var persistentKeepalive int64
+	noInternalIP := false
-	if keepAlive, ok := node.ObjectMeta.Annotations[persistentKeepaliveKey]; !ok {
+	if s, ok := node.ObjectMeta.Annotations[forceInternalIPAnnotationKey]; ok && (s == "" || s == "-") {
-		persistentKeepalive = 0
+		noInternalIP = true
-	} else {
+		internalIP = nil
-		if persistentKeepalive, err = strconv.ParseInt(keepAlive, 10, 64); err != nil {
-			persistentKeepalive = 0
 	}
+	// Set Wireguard PersistentKeepalive setting for the node.
+	var persistentKeepalive time.Duration
+	if keepAlive, ok := node.ObjectMeta.Annotations[persistentKeepaliveKey]; ok {
+		// We can ignore the error, because p will be set to 0 if an error occures.
+		p, _ := strconv.ParseInt(keepAlive, 10, 64)
+		persistentKeepalive = time.Duration(p) * time.Second
 	}
 	var lastSeen int64
 	if ls, ok := node.ObjectMeta.Annotations[lastSeenAnnotationKey]; !ok {
@@ -282,24 +312,60 @@ func translateNode(node *v1.Node) *mesh.Node {
 			lastSeen = 0
 		}
 	}
+	var discoveredEndpoints map[string]*net.UDPAddr
+	if de, ok := node.ObjectMeta.Annotations[discoveredEndpointsKey]; ok {
+		err := json.Unmarshal([]byte(de), &discoveredEndpoints)
+		if err != nil {
+			discoveredEndpoints = nil
+		}
+	}
+	// Set allowed IPs for a location.
+	var allowedLocationIPs []net.IPNet
+	if str, ok := node.ObjectMeta.Annotations[allowedLocationIPsKey]; ok {
+		for _, ip := range strings.Split(str, ",") {
+			if ipnet := normalizeIP(ip); ipnet != nil {
+				allowedLocationIPs = append(allowedLocationIPs, *ipnet)
+			}
+		}
+	}
+	var meshGranularity mesh.Granularity
+	if gr, ok := node.ObjectMeta.Annotations[granularityKey]; ok {
+		meshGranularity = mesh.Granularity(gr)
+		switch meshGranularity {
+		case mesh.LogicalGranularity:
+		case mesh.FullGranularity:
+		default:
+			meshGranularity = ""
+		}
+	}
+
+	// TODO log some error or warning.
+	key, _ := wgtypes.ParseKey(node.ObjectMeta.Annotations[keyAnnotationKey])
+
 	return &mesh.Node{
 		// Endpoint and InternalIP should only ever fail to parse if the
 		// remote node's agent has not yet set its IP address;
 		// in this case the IP will be nil and
 		// the mesh can wait for the node to be updated.
+		// It is valid for the InternalIP to be nil,
+		// if the given node only has public IP addresses.
 		Endpoint:            endpoint,
+		NoInternalIP:        noInternalIP,
 		InternalIP:          internalIP,
-		Key:                 []byte(node.ObjectMeta.Annotations[keyAnnotationKey]),
+		Key:                 key,
 		LastSeen:            lastSeen,
 		Leader:              leader,
 		Location:            location,
 		Name:                node.Name,
-		PersistentKeepalive: int(persistentKeepalive),
+		PersistentKeepalive: persistentKeepalive,
 		Subnet:              subnet,
 		// WireGuardIP can fail to parse if the node is not a leader or if
 		// the node's agent has not yet reconciled. In either case, the IP
 		// will parse as nil.
 		WireGuardIP:         normalizeIP(node.ObjectMeta.Annotations[wireGuardIPAnnotationKey]),
+		DiscoveredEndpoints: discoveredEndpoints,
+		AllowedLocationIPs:  allowedLocationIPs,
+		Granularity:         meshGranularity,
 	}
 }
 
@@ -308,14 +374,14 @@ func translatePeer(peer *v1alpha1.Peer) *mesh.Peer {
 	if peer == nil {
 		return nil
 	}
-	var aips []*net.IPNet
+	var aips []net.IPNet
 	for _, aip := range peer.Spec.AllowedIPs {
 		aip := normalizeIP(aip)
 		// Skip any invalid IPs.
 		if aip == nil {
 			continue
 		}
-		aips = append(aips, aip)
+		aips = append(aips, *aip)
 	}
 	var endpoint *wireguard.Endpoint
 	if peer.Spec.Endpoint != nil {
@@ -325,39 +391,47 @@ func translatePeer(peer *v1alpha1.Peer) *mesh.Peer {
 		} else {
 			ip = ip.To16()
 		}
-		if peer.Spec.Endpoint.Port > 0 && ip != nil {
+		if peer.Spec.Endpoint.Port > 0 {
-			endpoint = &wireguard.Endpoint{
+			if ip != nil {
-				DNSOrIP: wireguard.DNSOrIP{IP: ip},
+				endpoint = wireguard.NewEndpoint(ip, int(peer.Spec.Endpoint.Port))
-				Port:    peer.Spec.Endpoint.Port,
+			}
+			if peer.Spec.Endpoint.DNS != "" {
+				endpoint = wireguard.ParseEndpoint(fmt.Sprintf("%s:%d", peer.Spec.Endpoint.DNS, peer.Spec.Endpoint.Port))
 			}
 		}
 	}
-	var key []byte
+
-	if len(peer.Spec.PublicKey) > 0 {
+	key, err := wgtypes.ParseKey(peer.Spec.PublicKey)
-		key = []byte(peer.Spec.PublicKey)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to parse public key", "peer", peer.Name, "err", err.Error())
 	}
-	var psk []byte
+	var psk *wgtypes.Key
-	if len(peer.Spec.PresharedKey) > 0 {
+	if k, err := wgtypes.ParseKey(peer.Spec.PresharedKey); err != nil {
-		psk = []byte(peer.Spec.PresharedKey)
+		// Set key to nil to avoid setting a key to the zero value wgtypes.Key{}
+		psk = nil
+	} else {
+		psk = &k
 	}
-	var pka int
+	var pka time.Duration
 	if peer.Spec.PersistentKeepalive > 0 {
-		pka = peer.Spec.PersistentKeepalive
+		pka = time.Duration(peer.Spec.PersistentKeepalive) * time.Second
 	}
 	return &mesh.Peer{
 		Name: peer.Name,
 		Peer: wireguard.Peer{
+			PeerConfig: wgtypes.PeerConfig{
 				AllowedIPs:                  aips,
-			Endpoint:            endpoint,
+				PersistentKeepaliveInterval: &pka,
-			PersistentKeepalive: pka,
 				PresharedKey:                psk,
 				PublicKey:                   key,
 			},
+			Endpoint: endpoint,
+		},
 	}
 }
 
 // CleanUp removes configuration applied to the backend.
-func (pb *peerBackend) CleanUp(name string) error {
+func (pb *peerBackend) CleanUp(_ context.Context, _ string) error {
 	return nil
 }
 
@@ -372,29 +446,14 @@ func (pb *peerBackend) Get(name string) (*mesh.Peer, error) {
 
 // Init initializes the backend; for this backend that means
 // syncing the informer cache.
-func (pb *peerBackend) Init(stop <-chan struct{}) error {
+func (pb *peerBackend) Init(ctx context.Context) error {
-	// Register CRD.
+	// Check the presents of the CRD peers.kilo.squat.ai.
-	crd := crdutils.NewCustomResourceDefinition(crdutils.Config{
+	if _, err := pb.extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, strings.Join([]string{v1alpha1.PeerPlural, v1alpha1.GroupName}, "."), metav1.GetOptions{}); err != nil {
-		SpecDefinitionName:    "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1.Peer",
+		return fmt.Errorf("CRD is not present: %v", err)
-		EnableValidation:      true,
-		ResourceScope:         string(v1beta1.ClusterScoped),
-		Group:                 v1alpha1.GroupName,
-		Kind:                  v1alpha1.PeerKind,
-		Version:               v1alpha1.SchemeGroupVersion.Version,
-		Plural:                v1alpha1.PeerPlural,
-		ShortNames:            v1alpha1.PeerShortNames,
-		GetOpenAPIDefinitions: v1alpha1.GetOpenAPIDefinitions,
-	})
-	crd.Spec.Subresources.Scale = nil
-	crd.Spec.Subresources.Status = nil
-
-	_, err := pb.extensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
-	if err != nil && !apierrors.IsAlreadyExists(err) {
-		return fmt.Errorf("failed to create CRD: %v", err)
 	}
 
-	go pb.informer.Run(stop)
+	go pb.informer.Run(ctx.Done())
-	if ok := cache.WaitForCacheSync(stop, func() bool {
+	if ok := cache.WaitForCacheSync(ctx.Done(), func() bool {
 		return pb.informer.HasSynced()
 	}); !ok {
 		return errors.New("failed to sync peer cache")
@@ -453,7 +512,7 @@ func (pb *peerBackend) List() ([]*mesh.Peer, error) {
 }
 
 // Set sets the fields of a peer.
-func (pb *peerBackend) Set(name string, peer *mesh.Peer) error {
+func (pb *peerBackend) Set(ctx context.Context, name string, peer *mesh.Peer) error {
 	old, err := pb.lister.Get(name)
 	if err != nil {
 		return fmt.Errorf("failed to find peer: %v", err)
@@ -465,14 +524,25 @@ func (pb *peerBackend) Set(name string, peer *mesh.Peer) error {
 	}
 	if peer.Endpoint != nil {
 		p.Spec.Endpoint = &v1alpha1.PeerEndpoint{
-			IP:   peer.Endpoint.IP.String(),
+			DNSOrIP: v1alpha1.DNSOrIP{
-			Port: peer.Endpoint.Port,
+				IP:  peer.Endpoint.IP().String(),
+				DNS: peer.Endpoint.DNS(),
+			},
+			Port: uint32(peer.Endpoint.Port()),
 		}
 	}
-	p.Spec.PersistentKeepalive = peer.PersistentKeepalive
+	if peer.PersistentKeepaliveInterval == nil {
-	p.Spec.PresharedKey = string(peer.PresharedKey)
+		p.Spec.PersistentKeepalive = 0
-	p.Spec.PublicKey = string(peer.PublicKey)
+	} else {
-	if _, err = pb.client.KiloV1alpha1().Peers().Update(p); err != nil {
+		p.Spec.PersistentKeepalive = int(*peer.PersistentKeepaliveInterval / time.Second)
+	}
+	if peer.PresharedKey == nil {
+		p.Spec.PresharedKey = ""
+	} else {
+		p.Spec.PresharedKey = peer.PresharedKey.String()
+	}
+	p.Spec.PublicKey = peer.PublicKey.String()
+	if _, err = pb.client.KiloV1alpha1().Peers().Update(ctx, p, metav1.UpdateOptions{}); err != nil {
 		return fmt.Errorf("failed to update peer: %v", err)
 	}
 	return nil
@@ -495,35 +565,3 @@ func normalizeIP(ip string) *net.IPNet {
 	ipNet.IP = i.To16()
 	return ipNet
 }
-
-func parseEndpoint(endpoint string) *wireguard.Endpoint {
-	if len(endpoint) == 0 {
-		return nil
-	}
-	parts := strings.Split(endpoint, ":")
-	if len(parts) < 2 {
-		return nil
-	}
-	portRaw := parts[len(parts)-1]
-	hostRaw := strings.Trim(strings.Join(parts[:len(parts)-1], ":"), "[]")
-	port, err := strconv.ParseUint(portRaw, 10, 32)
-	if err != nil {
-		return nil
-	}
-	if len(validation.IsValidPortNum(int(port))) != 0 {
-		return nil
-	}
-	ip := net.ParseIP(hostRaw)
-	if ip == nil {
-		if len(validation.IsDNS1123Subdomain(hostRaw)) == 0 {
-			return &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{DNS: hostRaw}, Port: uint32(port)}
-		}
-		return nil
-	}
-	if ip4 := ip.To4(); ip4 != nil {
-		ip = ip4
-	} else {
-		ip = ip.To16()
-	}
-	return &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: ip}, Port: uint32(port)}
-}

pkg/k8s/backend_test.go

@@ -1,4 +1,4 @@
-// Copyright 2019 the Kilo authors
+// Copyright 2021 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,8 +17,10 @@ package k8s
 import (
 	"net"
 	"testing"
+	"time"
 
 	"github.com/kylelemons/godebug/pretty"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
@@ -26,6 +28,30 @@ import (
 	"github.com/squat/kilo/pkg/wireguard"
 )
 
+func mustKey() (k wgtypes.Key) {
+	var err error
+	if k, err = wgtypes.GeneratePrivateKey(); err != nil {
+		panic(err.Error())
+	}
+	return
+}
+
+func mustPSKKey() (key *wgtypes.Key) {
+	if k, err := wgtypes.GenerateKey(); err != nil {
+		panic(err.Error())
+	} else {
+		key = &k
+	}
+	return
+}
+
+var (
+	fooKey = mustKey()
+	pskKey = mustPSKKey()
+	second = time.Second
+	zero   = time.Duration(0)
+)
+
 func TestTranslateNode(t *testing.T) {
 	for _, tc := range []struct {
 		name        string
@@ -54,8 +80,19 @@ func TestTranslateNode(t *testing.T) {
 				internalIPAnnotationKey: "10.0.0.2/32",
 			},
 			out: &mesh.Node{
-				Endpoint:   &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: mesh.DefaultKiloPort},
+				Endpoint:   wireguard.NewEndpoint(net.ParseIP("10.0.0.1").To4(), mesh.DefaultKiloPort),
-				InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
+				InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2").To4(), Mask: net.CIDRMask(32, 32)},
+			},
+		},
+		{
+			name: "valid ips with ipv6",
+			annotations: map[string]string{
+				endpointAnnotationKey:   "[ff10::10]:51820",
+				internalIPAnnotationKey: "ff60::10/64",
+			},
+			out: &mesh.Node{
+				Endpoint:   wireguard.NewEndpoint(net.ParseIP("ff10::10").To16(), mesh.DefaultKiloPort),
+				InternalIP: &net.IPNet{IP: net.ParseIP("ff60::10").To16(), Mask: net.CIDRMask(64, 128)},
 			},
 		},
 		{
@@ -68,7 +105,7 @@ func TestTranslateNode(t *testing.T) {
 			name:        "normalize subnet",
 			annotations: map[string]string{},
 			out: &mesh.Node{
-				Subnet: &net.IPNet{IP: net.ParseIP("10.2.0.0"), Mask: net.CIDRMask(24, 32)},
+				Subnet: &net.IPNet{IP: net.ParseIP("10.2.0.0").To4(), Mask: net.CIDRMask(24, 32)},
 			},
 			subnet: "10.2.0.1/24",
 		},
@@ -76,14 +113,14 @@ func TestTranslateNode(t *testing.T) {
 			name:        "valid subnet",
 			annotations: map[string]string{},
 			out: &mesh.Node{
-				Subnet: &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				Subnet: &net.IPNet{IP: net.ParseIP("10.2.1.0").To4(), Mask: net.CIDRMask(24, 32)},
 			},
 			subnet: "10.2.1.0/24",
 		},
 		{
 			name: "region",
 			labels: map[string]string{
-				regionLabelKey: "a",
+				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
 				Location: "a",
@@ -95,7 +132,7 @@ func TestTranslateNode(t *testing.T) {
 				locationAnnotationKey: "b",
 			},
 			labels: map[string]string{
-				regionLabelKey: "a",
+				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
 				Location: "b",
@@ -108,7 +145,7 @@ func TestTranslateNode(t *testing.T) {
 				forceEndpointAnnotationKey: "-10.0.0.2:51821",
 			},
 			out: &mesh.Node{
-				Endpoint: &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: mesh.DefaultKiloPort},
+				Endpoint: wireguard.NewEndpoint(net.ParseIP("10.0.0.1").To4(), mesh.DefaultKiloPort),
 			},
 		},
 		{
@@ -118,7 +155,7 @@ func TestTranslateNode(t *testing.T) {
 				forceEndpointAnnotationKey: "10.0.0.2:51821",
 			},
 			out: &mesh.Node{
-				Endpoint: &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.2")}, Port: 51821},
+				Endpoint: wireguard.NewEndpoint(net.ParseIP("10.0.0.2").To4(), 51821),
 			},
 		},
 		{
@@ -127,7 +164,7 @@ func TestTranslateNode(t *testing.T) {
 				persistentKeepaliveKey: "25",
 			},
 			out: &mesh.Node{
-				PersistentKeepalive: 25,
+				PersistentKeepalive: 25 * time.Second,
 			},
 		},
 		{
@@ -137,7 +174,8 @@ func TestTranslateNode(t *testing.T) {
 				forceInternalIPAnnotationKey: "-10.1.0.2/24",
 			},
 			out: &mesh.Node{
-				InternalIP: &net.IPNet{IP: net.ParseIP("10.1.0.1"), Mask: net.CIDRMask(24, 32)},
+				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.1").To4(), Mask: net.CIDRMask(24, 32)},
+				NoInternalIP: false,
 			},
 		},
 		{
@@ -147,7 +185,8 @@ func TestTranslateNode(t *testing.T) {
 				forceInternalIPAnnotationKey: "10.1.0.2/24",
 			},
 			out: &mesh.Node{
-				InternalIP: &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(24, 32)},
+				InternalIP:   &net.IPNet{IP: net.ParseIP("10.1.0.2").To4(), Mask: net.CIDRMask(24, 32)},
+				NoInternalIP: false,
 			},
 		},
 		{
@@ -164,7 +203,7 @@ func TestTranslateNode(t *testing.T) {
 				forceEndpointAnnotationKey:   "10.0.0.2:51821",
 				forceInternalIPAnnotationKey: "10.1.0.2/32",
 				internalIPAnnotationKey:      "10.1.0.1/32",
-				keyAnnotationKey:             "foo",
+				keyAnnotationKey:             fooKey.String(),
 				lastSeenAnnotationKey:        "1000000000",
 				leaderAnnotationKey:          "",
 				locationAnnotationKey:        "b",
@@ -172,16 +211,104 @@ func TestTranslateNode(t *testing.T) {
 				wireGuardIPAnnotationKey:     "10.4.0.1/16",
 			},
 			labels: map[string]string{
-				regionLabelKey: "a",
+				RegionLabelKey: "a",
 			},
 			out: &mesh.Node{
-				Endpoint:            &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.2")}, Port: 51821},
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("10.0.0.2").To4(), 51821),
-				InternalIP:          &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(32, 32)},
+				NoInternalIP:        false,
-				Key:                 []byte("foo"),
+				InternalIP:          &net.IPNet{IP: net.ParseIP("10.1.0.2").To4(), Mask: net.CIDRMask(32, 32)},
+				Key:                 fooKey,
 				LastSeen:            1000000000,
 				Leader:              true,
 				Location:            "b",
-				PersistentKeepalive: 25,
+				PersistentKeepalive: 25 * time.Second,
+				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0").To4(), Mask: net.CIDRMask(24, 32)},
+				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1").To4(), Mask: net.CIDRMask(16, 32)},
+			},
+			subnet: "10.2.1.0/24",
+		},
+		{
+			name: "complete with ipv6",
+			annotations: map[string]string{
+				endpointAnnotationKey:        "10.0.0.1:51820",
+				forceEndpointAnnotationKey:   "[1100::10]:51821",
+				forceInternalIPAnnotationKey: "10.1.0.2/32",
+				internalIPAnnotationKey:      "10.1.0.1/32",
+				keyAnnotationKey:             fooKey.String(),
+				lastSeenAnnotationKey:        "1000000000",
+				leaderAnnotationKey:          "",
+				locationAnnotationKey:        "b",
+				persistentKeepaliveKey:       "25",
+				wireGuardIPAnnotationKey:     "10.4.0.1/16",
+			},
+			labels: map[string]string{
+				RegionLabelKey: "a",
+			},
+			out: &mesh.Node{
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("1100::10"), 51821),
+				NoInternalIP:        false,
+				InternalIP:          &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(32, 32)},
+				Key:                 fooKey,
+				LastSeen:            1000000000,
+				Leader:              true,
+				Location:            "b",
+				PersistentKeepalive: 25 * time.Second,
+				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
+			},
+			subnet: "10.2.1.0/24",
+		},
+		{
+			name: "no InternalIP",
+			annotations: map[string]string{
+				endpointAnnotationKey:    "10.0.0.1:51820",
+				internalIPAnnotationKey:  "",
+				keyAnnotationKey:         fooKey.String(),
+				lastSeenAnnotationKey:    "1000000000",
+				locationAnnotationKey:    "b",
+				persistentKeepaliveKey:   "25",
+				wireGuardIPAnnotationKey: "10.4.0.1/16",
+			},
+			labels: map[string]string{
+				RegionLabelKey: "a",
+			},
+			out: &mesh.Node{
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("10.0.0.1"), 51820),
+				InternalIP:          nil,
+				Key:                 fooKey,
+				LastSeen:            1000000000,
+				Leader:              false,
+				Location:            "b",
+				PersistentKeepalive: 25 * time.Second,
+				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
+			},
+			subnet: "10.2.1.0/24",
+		},
+		{
+			name: "Force no internal IP",
+			annotations: map[string]string{
+				endpointAnnotationKey:        "10.0.0.1:51820",
+				internalIPAnnotationKey:      "10.1.0.1/32",
+				forceInternalIPAnnotationKey: "",
+				keyAnnotationKey:             fooKey.String(),
+				lastSeenAnnotationKey:        "1000000000",
+				locationAnnotationKey:        "b",
+				persistentKeepaliveKey:       "25",
+				wireGuardIPAnnotationKey:     "10.4.0.1/16",
+			},
+			labels: map[string]string{
+				RegionLabelKey: "a",
+			},
+			out: &mesh.Node{
+				Endpoint:            wireguard.NewEndpoint(net.ParseIP("10.0.0.1"), 51820),
+				NoInternalIP:        true,
+				InternalIP:          nil,
+				Key:                 fooKey,
+				LastSeen:            1000000000,
+				Leader:              false,
+				Location:            "b",
+				PersistentKeepalive: 25 * time.Second,
 				Subnet:              &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
 				WireGuardIP:         &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
 			},
@@ -192,7 +319,7 @@ func TestTranslateNode(t *testing.T) {
 		n.ObjectMeta.Annotations = tc.annotations
 		n.ObjectMeta.Labels = tc.labels
 		n.Spec.PodCIDR = tc.subnet
-		node := translateNode(n)
+		node := translateNode(n, RegionLabelKey)
 		if diff := pretty.Compare(node, tc.out); diff != "" {
 			t.Errorf("test case %q: got diff: %v", tc.name, diff)
 		}
@@ -207,7 +334,13 @@ func TestTranslatePeer(t *testing.T) {
 	}{
 		{
 			name: "empty",
-			out:  &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "invalid ips",
@@ -217,7 +350,13 @@ func TestTranslatePeer(t *testing.T) {
 					"foo",
 				},
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "valid ips",
@@ -229,10 +368,13 @@ func TestTranslatePeer(t *testing.T) {
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					AllowedIPs: []*net.IPNet{
+					PeerConfig: wgtypes.PeerConfig{
+						AllowedIPs: []net.IPNet{
 							{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(24, 32)},
 							{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
 						},
+						PersistentKeepaliveInterval: &zero,
+					},
 				},
 			},
 		},
@@ -240,27 +382,90 @@ func TestTranslatePeer(t *testing.T) {
 			name: "invalid endpoint ip",
 			spec: v1alpha1.PeerSpec{
 				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
 						IP: "foo",
-					Port: mesh.DefaultKiloPort,
 					},
-			},
-			out: &mesh.Peer{},
-		},
-		{
-			name: "valid endpoint",
-			spec: v1alpha1.PeerSpec{
-				Endpoint: &v1alpha1.PeerEndpoint{
-					IP:   "10.0.0.1",
 					Port: mesh.DefaultKiloPort,
 				},
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					Endpoint: &wireguard.Endpoint{
+					PeerConfig: wgtypes.PeerConfig{
-						DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")},
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
+		},
+		{
+			name: "only endpoint port",
+			spec: v1alpha1.PeerSpec{
+				Endpoint: &v1alpha1.PeerEndpoint{
 					Port: mesh.DefaultKiloPort,
 				},
 			},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
+		},
+		{
+			name: "valid endpoint ip",
+			spec: v1alpha1.PeerSpec{
+				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
+						IP: "10.0.0.1",
+					},
+					Port: mesh.DefaultKiloPort,
+				},
+			},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+					Endpoint: wireguard.NewEndpoint(net.ParseIP("10.0.0.1").To4(), mesh.DefaultKiloPort),
+				},
+			},
+		},
+		{
+			name: "valid endpoint ipv6",
+			spec: v1alpha1.PeerSpec{
+				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
+						IP: "ff60::2",
+					},
+					Port: mesh.DefaultKiloPort,
+				},
+			},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+					Endpoint: wireguard.NewEndpoint(net.ParseIP("ff60::2").To16(), mesh.DefaultKiloPort),
+				},
+			},
+		},
+		{
+			name: "valid endpoint DNS",
+			spec: v1alpha1.PeerSpec{
+				Endpoint: &v1alpha1.PeerEndpoint{
+					DNSOrIP: v1alpha1.DNSOrIP{
+						DNS: "example.com",
+					},
+					Port: mesh.DefaultKiloPort,
+				},
+			},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					Endpoint: wireguard.ParseEndpoint("example.com:51820"),
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
 			},
 		},
 		{
@@ -268,16 +473,25 @@ func TestTranslatePeer(t *testing.T) {
 			spec: v1alpha1.PeerSpec{
 				PublicKey: "",
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "valid key",
 			spec: v1alpha1.PeerSpec{
-				PublicKey: "foo",
+				PublicKey: fooKey.String(),
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					PublicKey: []byte("foo"),
+					PeerConfig: wgtypes.PeerConfig{
+						PublicKey:                   fooKey,
+						PersistentKeepaliveInterval: &zero,
+					},
 				},
 			},
 		},
@@ -286,7 +500,13 @@ func TestTranslatePeer(t *testing.T) {
 			spec: v1alpha1.PeerSpec{
 				PersistentKeepalive: -1,
 			},
-			out: &mesh.Peer{},
+			out: &mesh.Peer{
+				Peer: wireguard.Peer{
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+					},
+				},
+			},
 		},
 		{
 			name: "valid keepalive",
@@ -295,18 +515,23 @@ func TestTranslatePeer(t *testing.T) {
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					PersistentKeepalive: 1,
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &second,
+					},
 				},
 			},
 		},
 		{
 			name: "valid preshared key",
 			spec: v1alpha1.PeerSpec{
-				PresharedKey: "psk",
+				PresharedKey: pskKey.String(),
 			},
 			out: &mesh.Peer{
 				Peer: wireguard.Peer{
-					PresharedKey: []byte("psk"),
+					PeerConfig: wgtypes.PeerConfig{
+						PersistentKeepaliveInterval: &zero,
+						PresharedKey:                pskKey,
+					},
 				},
 			},
 		},
@@ -319,52 +544,3 @@ func TestTranslatePeer(t *testing.T) {
 		}
 	}
 }
-
-func TestParseEndpoint(t *testing.T) {
-	for _, tc := range []struct {
-		name     string
-		endpoint string
-		out      *wireguard.Endpoint
-	}{
-		{
-			name:     "empty",
-			endpoint: "",
-			out:      nil,
-		},
-		{
-			name:     "invalid IP",
-			endpoint: "10.0.0.:51820",
-			out:      nil,
-		},
-		{
-			name:     "invalid hostname",
-			endpoint: "foo-:51820",
-			out:      nil,
-		},
-		{
-			name:     "invalid port",
-			endpoint: "10.0.0.1:100000000",
-			out:      nil,
-		},
-		{
-			name:     "valid IP",
-			endpoint: "10.0.0.1:51820",
-			out:      &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("10.0.0.1")}, Port: mesh.DefaultKiloPort},
-		},
-		{
-			name:     "valid IPv6",
-			endpoint: "[ff02::114]:51820",
-			out:      &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{IP: net.ParseIP("ff02::114")}, Port: mesh.DefaultKiloPort},
-		},
-		{
-			name:     "valid hostname",
-			endpoint: "foo:51821",
-			out:      &wireguard.Endpoint{DNSOrIP: wireguard.DNSOrIP{DNS: "foo"}, Port: 51821},
-		},
-	} {
-		endpoint := parseEndpoint(tc.endpoint)
-		if diff := pretty.Compare(endpoint, tc.out); diff != "" {
-			t.Errorf("test case %q: got diff: %v", tc.name, diff)
-		}
-	}
-}

pkg/k8s/clientset/versioned/clientset.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,9 @@
 package versioned
 
 import (
+	"fmt"
+	"net/http"
+
 	kilov1alpha1 "github.com/squat/kilo/pkg/k8s/clientset/versioned/typed/kilo/v1alpha1"
 	discovery "k8s.io/client-go/discovery"
 	rest "k8s.io/client-go/rest"
@@ -49,19 +52,47 @@ func (c *Clientset) Discovery() discovery.DiscoveryInterface {
 }
 
 // NewForConfig creates a new Clientset for the given config.
+// If config's RateLimiter is not set and QPS and Burst are acceptable,
+// NewForConfig will generate a rate-limiter in configShallowCopy.
+// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
+// where httpClient was generated with rest.HTTPClientFor(c).
 func NewForConfig(c *rest.Config) (*Clientset, error) {
 	configShallowCopy := *c
-	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
+
-		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
+	if configShallowCopy.UserAgent == "" {
+		configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent()
 	}
-	var cs Clientset
+
-	var err error
+	// share the transport between all clients
-	cs.kiloV1alpha1, err = kilov1alpha1.NewForConfig(&configShallowCopy)
+	httpClient, err := rest.HTTPClientFor(&configShallowCopy)
 	if err != nil {
 		return nil, err
 	}
 
-	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)
+	return NewForConfigAndClient(&configShallowCopy, httpClient)
+}
+
+// NewForConfigAndClient creates a new Clientset for the given config and http client.
+// Note the http client provided takes precedence over the configured transport values.
+// If config's RateLimiter is not set and QPS and Burst are acceptable,
+// NewForConfigAndClient will generate a rate-limiter in configShallowCopy.
+func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) {
+	configShallowCopy := *c
+	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
+		if configShallowCopy.Burst <= 0 {
+			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
+		}
+		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
+	}
+
+	var cs Clientset
+	var err error
+	cs.kiloV1alpha1, err = kilov1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient)
+	if err != nil {
+		return nil, err
+	}
+
+	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient)
 	if err != nil {
 		return nil, err
 	}
@@ -71,11 +102,11 @@ func NewForConfig(c *rest.Config) (*Clientset, error) {
 // NewForConfigOrDie creates a new Clientset for the given config and
 // panics if there is an error in the config.
 func NewForConfigOrDie(c *rest.Config) *Clientset {
-	var cs Clientset
+	cs, err := NewForConfig(c)
-	cs.kiloV1alpha1 = kilov1alpha1.NewForConfigOrDie(c)
+	if err != nil {
-
+		panic(err)
-	cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c)
+	}
-	return &cs
+	return cs
 }
 
 // New creates a new Clientset for the given RESTClient.

pkg/k8s/clientset/versioned/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/fake/clientset_generated.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -39,7 +39,7 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset {
 		}
 	}
 
-	cs := &Clientset{}
+	cs := &Clientset{tracker: o}
 	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
 	cs.AddReactor("*", "*", testing.ObjectReaction(o))
 	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
@@ -61,13 +61,21 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset {
 type Clientset struct {
 	testing.Fake
 	discovery *fakediscovery.FakeDiscovery
+	tracker   testing.ObjectTracker
 }
 
 func (c *Clientset) Discovery() discovery.DiscoveryInterface {
 	return c.discovery
 }
 
-var _ clientset.Interface = &Clientset{}
+func (c *Clientset) Tracker() testing.ObjectTracker {
+	return c.tracker
+}
+
+var (
+	_ clientset.Interface = &Clientset{}
+	_ testing.FakeClient  = &Clientset{}
+)
 
 // KiloV1alpha1 retrieves the KiloV1alpha1Client
 func (c *Clientset) KiloV1alpha1() kilov1alpha1.KiloV1alpha1Interface {

pkg/k8s/clientset/versioned/fake/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/fake/register.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ import (
 
 var scheme = runtime.NewScheme()
 var codecs = serializer.NewCodecFactory(scheme)
-var parameterCodec = runtime.NewParameterCodec(scheme)
+
 var localSchemeBuilder = runtime.SchemeBuilder{
 	kilov1alpha1.AddToScheme,
 }

pkg/k8s/clientset/versioned/scheme/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/scheme/register.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/doc.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/fake_kilo_client.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/fake/fake_peer.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 package fake
 
 import (
+	"context"
+
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	labels "k8s.io/apimachinery/pkg/labels"
@@ -31,12 +33,12 @@ type FakePeers struct {
 	Fake *FakeKiloV1alpha1
 }
 
-var peersResource = schema.GroupVersionResource{Group: "kilo", Version: "v1alpha1", Resource: "peers"}
+var peersResource = schema.GroupVersionResource{Group: "kilo.squat.ai", Version: "v1alpha1", Resource: "peers"}
 
-var peersKind = schema.GroupVersionKind{Group: "kilo", Version: "v1alpha1", Kind: "Peer"}
+var peersKind = schema.GroupVersionKind{Group: "kilo.squat.ai", Version: "v1alpha1", Kind: "Peer"}
 
 // Get takes name of the peer, and returns the corresponding peer object, and an error if there is any.
-func (c *FakePeers) Get(name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootGetAction(peersResource, name), &v1alpha1.Peer{})
 	if obj == nil {
@@ -46,7 +48,7 @@ func (c *FakePeers) Get(name string, options v1.GetOptions) (result *v1alpha1.Pe
 }
 
 // List takes label and field selectors, and returns the list of Peers that match those selectors.
-func (c *FakePeers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
+func (c *FakePeers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootListAction(peersResource, peersKind, opts), &v1alpha1.PeerList{})
 	if obj == nil {
@@ -67,13 +69,13 @@ func (c *FakePeers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err er
 }
 
 // Watch returns a watch.Interface that watches the requested peers.
-func (c *FakePeers) Watch(opts v1.ListOptions) (watch.Interface, error) {
+func (c *FakePeers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
 	return c.Fake.
 		InvokesWatch(testing.NewRootWatchAction(peersResource, opts))
 }
 
 // Create takes the representation of a peer and creates it.  Returns the server's representation of the peer, and an error, if there is any.
-func (c *FakePeers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootCreateAction(peersResource, peer), &v1alpha1.Peer{})
 	if obj == nil {
@@ -83,7 +85,7 @@ func (c *FakePeers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err erro
 }
 
 // Update takes the representation of a peer and updates it. Returns the server's representation of the peer, and an error, if there is any.
-func (c *FakePeers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootUpdateAction(peersResource, peer), &v1alpha1.Peer{})
 	if obj == nil {
@@ -93,22 +95,22 @@ func (c *FakePeers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err erro
 }
 
 // Delete takes name of the peer and deletes it. Returns an error if one occurs.
-func (c *FakePeers) Delete(name string, options *v1.DeleteOptions) error {
+func (c *FakePeers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	_, err := c.Fake.
-		Invokes(testing.NewRootDeleteAction(peersResource, name), &v1alpha1.Peer{})
+		Invokes(testing.NewRootDeleteActionWithOptions(peersResource, name, opts), &v1alpha1.Peer{})
 	return err
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *FakePeers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+func (c *FakePeers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
-	action := testing.NewRootDeleteCollectionAction(peersResource, listOptions)
+	action := testing.NewRootDeleteCollectionAction(peersResource, listOpts)
 
 	_, err := c.Fake.Invokes(action, &v1alpha1.PeerList{})
 	return err
 }
 
 // Patch applies the patch and returns the patched peer.
-func (c *FakePeers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error) {
+func (c *FakePeers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error) {
 	obj, err := c.Fake.
 		Invokes(testing.NewRootPatchSubresourceAction(peersResource, name, pt, data, subresources...), &v1alpha1.Peer{})
 	if obj == nil {

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/generated_expansion.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/kilo_client.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,9 +17,10 @@
 package v1alpha1
 
 import (
+	"net/http"
+
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
 	"github.com/squat/kilo/pkg/k8s/clientset/versioned/scheme"
-	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
 	rest "k8s.io/client-go/rest"
 )
 
@@ -28,7 +29,7 @@ type KiloV1alpha1Interface interface {
 	PeersGetter
 }
 
-// KiloV1alpha1Client is used to interact with features provided by the kilo group.
+// KiloV1alpha1Client is used to interact with features provided by the kilo.squat.ai group.
 type KiloV1alpha1Client struct {
 	restClient rest.Interface
 }
@@ -38,12 +39,28 @@ func (c *KiloV1alpha1Client) Peers() PeerInterface {
 }
 
 // NewForConfig creates a new KiloV1alpha1Client for the given config.
+// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
+// where httpClient was generated with rest.HTTPClientFor(c).
 func NewForConfig(c *rest.Config) (*KiloV1alpha1Client, error) {
 	config := *c
 	if err := setConfigDefaults(&config); err != nil {
 		return nil, err
 	}
-	client, err := rest.RESTClientFor(&config)
+	httpClient, err := rest.HTTPClientFor(&config)
+	if err != nil {
+		return nil, err
+	}
+	return NewForConfigAndClient(&config, httpClient)
+}
+
+// NewForConfigAndClient creates a new KiloV1alpha1Client for the given config and http client.
+// Note the http client provided takes precedence over the configured transport values.
+func NewForConfigAndClient(c *rest.Config, h *http.Client) (*KiloV1alpha1Client, error) {
+	config := *c
+	if err := setConfigDefaults(&config); err != nil {
+		return nil, err
+	}
+	client, err := rest.RESTClientForConfigAndClient(&config, h)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +86,7 @@ func setConfigDefaults(config *rest.Config) error {
 	gv := v1alpha1.SchemeGroupVersion
 	config.GroupVersion = &gv
 	config.APIPath = "/apis"
-	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
+	config.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
 
 	if config.UserAgent == "" {
 		config.UserAgent = rest.DefaultKubernetesUserAgent()

pkg/k8s/clientset/versioned/typed/kilo/v1alpha1/peer.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package v1alpha1
 
 import (
+	"context"
 	"time"
 
 	v1alpha1 "github.com/squat/kilo/pkg/k8s/apis/kilo/v1alpha1"
@@ -35,14 +36,14 @@ type PeersGetter interface {
 
 // PeerInterface has methods to work with Peer resources.
 type PeerInterface interface {
-	Create(*v1alpha1.Peer) (*v1alpha1.Peer, error)
+	Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (*v1alpha1.Peer, error)
-	Update(*v1alpha1.Peer) (*v1alpha1.Peer, error)
+	Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (*v1alpha1.Peer, error)
-	Delete(name string, options *v1.DeleteOptions) error
+	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
-	DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
+	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
-	Get(name string, options v1.GetOptions) (*v1alpha1.Peer, error)
+	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.Peer, error)
-	List(opts v1.ListOptions) (*v1alpha1.PeerList, error)
+	List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.PeerList, error)
-	Watch(opts v1.ListOptions) (watch.Interface, error)
+	Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error)
-	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error)
 	PeerExpansion
 }
 
@@ -59,19 +60,19 @@ func newPeers(c *KiloV1alpha1Client) *peers {
 }
 
 // Get takes name of the peer, and returns the corresponding peer object, and an error if there is any.
-func (c *peers) Get(name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
+func (c *peers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Get().
 		Resource("peers").
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // List takes label and field selectors, and returns the list of Peers that match those selectors.
-func (c *peers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
+func (c *peers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PeerList, err error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -81,13 +82,13 @@ func (c *peers) List(opts v1.ListOptions) (result *v1alpha1.PeerList, err error)
 		Resource("peers").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Watch returns a watch.Interface that watches the requested peers.
-func (c *peers) Watch(opts v1.ListOptions) (watch.Interface, error) {
+func (c *peers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
 	var timeout time.Duration
 	if opts.TimeoutSeconds != nil {
 		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
@@ -97,66 +98,69 @@ func (c *peers) Watch(opts v1.ListOptions) (watch.Interface, error) {
 		Resource("peers").
 		VersionedParams(&opts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Watch()
+		Watch(ctx)
 }
 
 // Create takes the representation of a peer and creates it.  Returns the server's representation of the peer, and an error, if there is any.
-func (c *peers) Create(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *peers) Create(ctx context.Context, peer *v1alpha1.Peer, opts v1.CreateOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Post().
 		Resource("peers").
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(peer).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Update takes the representation of a peer and updates it. Returns the server's representation of the peer, and an error, if there is any.
-func (c *peers) Update(peer *v1alpha1.Peer) (result *v1alpha1.Peer, err error) {
+func (c *peers) Update(ctx context.Context, peer *v1alpha1.Peer, opts v1.UpdateOptions) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Put().
 		Resource("peers").
 		Name(peer.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(peer).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }
 
 // Delete takes name of the peer and deletes it. Returns an error if one occurs.
-func (c *peers) Delete(name string, options *v1.DeleteOptions) error {
+func (c *peers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	return c.client.Delete().
 		Resource("peers").
 		Name(name).
-		Body(options).
+		Body(&opts).
-		Do().
+		Do(ctx).
 		Error()
 }
 
 // DeleteCollection deletes a collection of objects.
-func (c *peers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+func (c *peers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
 	var timeout time.Duration
-	if listOptions.TimeoutSeconds != nil {
+	if listOpts.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
 	}
 	return c.client.Delete().
 		Resource("peers").
-		VersionedParams(&listOptions, scheme.ParameterCodec).
+		VersionedParams(&listOpts, scheme.ParameterCodec).
 		Timeout(timeout).
-		Body(options).
+		Body(&opts).
-		Do().
+		Do(ctx).
 		Error()
 }
 
 // Patch applies the patch and returns the patched peer.
-func (c *peers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Peer, err error) {
+func (c *peers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Peer, err error) {
 	result = &v1alpha1.Peer{}
 	err = c.client.Patch(pt).
 		Resource("peers").
-		SubResource(subresources...).
 		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
 		Body(data).
-		Do().
+		Do(ctx).
 		Into(result)
 	return
 }

pkg/k8s/informers/factory.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/generic.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ func (f *genericInformer) Lister() cache.GenericLister {
 // TODO extend this to unknown resources with a client pool
 func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {
 	switch resource {
-	// Group=kilo, Version=v1alpha1
+	// Group=kilo.squat.ai, Version=v1alpha1
 	case v1alpha1.SchemeGroupVersion.WithResource("peers"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Kilo().V1alpha1().Peers().Informer()}, nil
 

pkg/k8s/informers/internalinterfaces/factory_interfaces.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

pkg/k8s/informers/kilo/interface.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Kilo authors
+// Copyright 2022 the Kilo authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.