Merge pull request #191 from squat/test_peers

e2e: test VPN in e2e tests

Makefile

@@ -207,8 +207,8 @@ $(BASH_UNIT):
 	curl -Lo $@ https://raw.githubusercontent.com/pgrange/bash_unit/v1.7.2/bash_unit
 	chmod +x $@
 
-e2e: container ${KIND_BINARY} ${KUBECTL_BINARY} $(BASH_UNIT)
+e2e: container $(KIND_BINARY) $(KUBECTL_BINARY) $(BASH_UNIT) bin/$(OS)/$(ARCH)/kgctl
-	KILO_IMAGE=${IMAGE}:${ARCH}-${VERSION} KIND_BINARY=${KIND_BINARY} $(BASH_UNIT) ./e2e/kind.sh
+	KILO_IMAGE=$(IMAGE):$(ARCH)-$(VERSION) KIND_BINARY=$(KIND_BINARY) KUBECTL_BINARY=$(KUBECTL_BINARY) KGCTL_BINARY=$(shell pwd)/bin/$(OS)/$(ARCH)/kgctl $(BASH_UNIT) ./e2e/kind.sh
 
 header: .header
 	@HEADER=$$(cat .header); \

e2e/kilo-kind-userspace.yaml

@@ -108,6 +108,7 @@ spec:
         - --hostname=$(NODE_NAME)
         - --create-interface=false
         - --mesh-granularity=full
+        - --kubeconfig=/etc/kubernetes/kubeconfig
         env:
         - name: NODE_NAME
           valueFrom:
@@ -132,6 +133,9 @@ spec:
         - name: wireguard
           mountPath: /var/run/wireguard
           readOnly: false
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
       - name: boringtun
         image: leonnicolas/boringtun:alpine
         args:
@@ -195,3 +199,6 @@ spec:
       - name: wireguard
         hostPath:
           path: /var/run/wireguard
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig

e2e/kind-config.yaml

@@ -7,3 +7,5 @@ nodes:
 networking:
   disableDefaultCNI: true # disable kindnet
   podSubnet: 10.42.0.0/16
+  apiServerAddress: 172.18.0.1
+  apiServerPort: 6443

e2e/kind.sh

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 # shellcheck disable=SC2034
-KUBECONFIG="kind.yaml"
+export KUBECONFIG="kind.yaml"
 KIND_CLUSTER="kind-cluster-kilo"
 KIND_BINARY="${KIND_BINARY:-kind}"
 KUBECTL_BINARY="${KUBECTL_BINARY:-kubectl}"
+KGCTL_BINARY="${KGCTL_BINARY:-kgctl}"
 KILO_IMAGE="${KILO_IMAGE:-squat/kilo}"
 
 retry() {
@@ -14,21 +15,47 @@ retry() {
 	shift 3
 	for c in $(seq 1 "$COUNT"); do
 		if "$@"; then
-		    return 0
+			return 0
 		else
 			printf "%s(attempt %d/%d)\n" "$ERROR" "$c" "$COUNT" | color "$YELLOW"
 			if [ "$c" != "$COUNT" ]; then
-			    printf "retrying in %d seconds...\n" "$SLEEP" | color "$YELLOW"
+				printf "retrying in %d seconds...\n" "$SLEEP" | color "$YELLOW"
-			    sleep "$SLEEP"
+				sleep "$SLEEP"
 			fi
 		fi
 	done
 	return 1
 }
 
+create_interface() {
+	docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun --foreground --disable-drop-privileges true "$1"
+}
+
+delete_interface() {
+	docker rm --force "$1"
+}
+
+create_peer() {
+	cat <<EOF | $KUBECTL_BINARY apply -f -
+apiVersion: kilo.squat.ai/v1alpha1
+kind: Peer
+metadata:
+  name: $1
+spec:
+  allowedIPs:
+  - $2
+  persistentKeepalive: $3
+  publicKey: $4
+EOF
+}
+
+delete_peer() {
+	$KUBECTL_BINARY delete peer "$1"
+}
+
 is_ready() {
 	for pod in $($KUBECTL_BINARY -n "$1" get pods -o name -l "$2"); do
-		if ! $KUBECTL_BINARY -n "$1" get "$pod" | tail -n 1 | grep -q  Running; then
+		if ! $KUBECTL_BINARY -n "$1" get "$pod" | tail -n 1 | grep -q Running; then
 			return 1;
 		fi
 	done
@@ -54,6 +81,8 @@ setup_suite() {
 	# Load the Kilo image into kind.
 	docker tag "$KILO_IMAGE" squat/kilo:test
 	$KIND_BINARY load docker-image squat/kilo:test --name $KIND_CLUSTER
+	# Create the kubeconfig secret.
+	$KUBECTL_BINARY create secret generic kubeconfig --from-file=kubeconfig="$KUBECONFIG" -n kube-system
 	# Apply Kilo the the cluster.
 	$KUBECTL_BINARY apply -f ../manifests/crds.yaml
 	$KUBECTL_BINARY apply -f kilo-kind-userspace.yaml
@@ -69,36 +98,68 @@ setup_suite() {
 }
 
 check_ping() {
-	    for ip in $($KUBECTL_BINARY get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+	local LOCAL
-		    ping=$($KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c "curl -s http://$ip:8080/ping")
+	while [ $# -gt 0 ]; do
-		    if [ "$ping" = "pong" ]; then
+		case $1 in
-			    echo "successfully pinged $ip"
+			--local)
-		    else
+			LOCAL=true
-			    printf 'failed to ping %s; expected "pong" but got "%s"\n' "$ip" "$ping"
+			;;
-			    return 1
+		esac
-		    fi
+		shift
-	    done
+	done
-	    return 0
+
+	for ip in $($KUBECTL_BINARY get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do
+		if [ -n "$LOCAL" ]; then
+			ping=$(curl -m 1 -s http://"$ip":8080/ping)
+		else
+			ping=$($KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c "curl -m 1 -s http://$ip:8080/ping")
+		fi
+		if [ "$ping" = "pong" ]; then
+			echo "successfully pinged $ip"
+		else
+			printf 'failed to ping %s; expected "pong" but got "%s"\n' "$ip" "$ping"
+			return 1
+		fi
+	done
+	return 0
 }
 
 check_adjacent() {
-	echo
+	$KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=fancy'
-	$KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -s adjacency:8080/?format=fancy'
 	assert_equals "12" \
-		"$($KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -s adjacency:8080/?format=json' | jq | grep -c true)" \
+		"$($KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=json' | jq | grep -c true)" \
 		"Adjacency returned the wrong number of successful pings"
 	echo "sleep for 30s (one reconciliation period) and try again..."
 	sleep 30
-	echo
+	$KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=fancy'
-	$KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -s adjacency:8080/?format=fancy'
 	assert_equals "12" \
-		"$($KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -s adjacency:8080/?format=json' | jq | grep -c true)" \
+		"$($KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=json' | jq | grep -c true)" \
 		 "Adjacency returned the wrong number of successful pings"
 }
 
+check_peer() {
+	local INTERFACE=$1
+	local PEER=$2
+	local ALLOWED_IP=$3
+	local GRANULARITY=$4
+	create_interface "$INTERFACE"
+	docker run --rm --entrypoint=/usr/bin/wg "$KILO_IMAGE" genkey > "$INTERFACE"
+	create_peer "$PEER" "$ALLOWED_IP" 10 "$(docker run --rm --entrypoint=/bin/sh -v "$PWD/$INTERFACE":/key "$KILO_IMAGE" -c 'cat /key | wg pubkey')"
+	$KGCTL_BINARY showconf peer "$PEER" --mesh-granularity="$GRANULARITY" > "$PEER".ini
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v "$PWD/$PEER.ini":/peer.ini "$KILO_IMAGE" setconf "$INTERFACE" /peer.ini
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v "$PWD/$INTERFACE":/key "$KILO_IMAGE" set "$INTERFACE" private-key /key
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" address add "$ALLOWED_IP" dev "$INTERFACE"
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" link set "$INTERFACE" up
+	docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" route add 10.42/16 dev "$INTERFACE"
+	retry 10 5 "" check_ping --local
+	rm "$INTERFACE" "$PEER".ini
+	delete_peer "$PEER"
+	delete_interface "$INTERFACE"
+}
+
 test_locationmesh() {
 	# shellcheck disable=SC2016
-	$KUBECTL_BINARY patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--mesh-granularity=location"]}]}}}}'
+	$KUBECTL_BINARY patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}'
 	sleep 5
 	block_until_ready_by_name kube-system kilo-userspace 
 	$KUBECTL_BINARY wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
@@ -107,9 +168,13 @@ test_locationmesh() {
 	retry 10 5 "the adjacency matrix is not complete yet" check_adjacent
 }
 
+test_locationmesh_peer() {
+	check_peer wg1 e2e 10.5.0.1/32 location
+}
+
 test_fullmesh() {
 	# shellcheck disable=SC2016
-	$KUBECTL_BINARY patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--mesh-granularity=full"]}]}}}}'
+	$KUBECTL_BINARY patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full"]}]}}}}'
 	sleep 5
 	block_until_ready_by_name kube-system kilo-userspace 
 	$KUBECTL_BINARY wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m
@@ -118,6 +183,10 @@ test_fullmesh() {
 	retry 10 5 "the adjacency matrix is not complete yet" check_adjacent
 }
 
+test_fullmesh_peer() {
+	check_peer wg1 e2e 10.5.0.1/32 full
+}
+
 teardown_suite () {
 	$KIND_BINARY delete clusters $KIND_CLUSTER
 }