manifests/: fix boringtun containers

A change in boringtun's cli caused the boringtun containers to crash.

Signed-off-by: leonnicolas <leonloechner@gmx.de>

.github/workflows/ci.yml

@@ -96,6 +96,7 @@ jobs:
       run: make unit
 
   e2e:
+    if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2

cmd/kg/main.go

@@ -245,11 +245,13 @@ func runRoot(_ *cobra.Command, _ []string) error {
 	if port < 1 || port > 1<<16-1 {
 		return fmt.Errorf("invalid port: port mus be in range [%d:%d], but got %d", 1, 1<<16-1, port)
 	}
-	m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, log.With(logger, "component", "kilo"), registry)
+	m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, log.With(logger, "component", "kilo"))
 	if err != nil {
 		return fmt.Errorf("failed to create Kilo mesh: %v", err)
 	}
 
+	m.RegisterMetrics(registry)
+
 	var g run.Group
 	{
 		h := internalserver.NewHandler(

e2e/kilo-kind-userspace.yaml

@@ -136,9 +136,9 @@ spec:
           mountPath: /etc/kubernetes
           readOnly: true
       - name: boringtun
-        image: leonnicolas/boringtun:cc19859
+        image: leonnicolas/boringtun
         args:
-        - --disable-drop-privileges=true
+        - --disable-drop-privileges
         - --foreground
         - kilo0
         securityContext:

e2e/lib.sh

@@ -65,7 +65,7 @@ build_kind_config() {
 }
 
 create_interface() {
-	docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun:cc19859 --foreground --disable-drop-privileges true "$1"
+	docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun --foreground --disable-drop-privileges "$1"
 }
 
 delete_interface() {

manifests/kilo-k3s-cilium.yaml

@@ -1,176 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kilo
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: kilo
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - list
-  - patch
-  - watch
-- apiGroups:
-  - kilo.squat.ai
-  resources:
-  - peers
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - apiextensions.k8s.io
-  resources:
-  - customresourcedefinitions
-  verbs:
-  - get
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kilo
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kilo
-subjects:
-  - kind: ServiceAccount
-    name: kilo
-    namespace: kube-system
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kilo-scripts
-  namespace: kube-system
-data:
-  init.sh: |
-    #!/bin/sh
-    cat > /etc/kubernetes/kubeconfig <<EOF
-        apiVersion: v1
-        kind: Config
-        name: kilo
-        clusters:
-        - cluster:
-            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
-            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
-        users:
-        - name: kilo
-          user:
-            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
-        contexts:
-        - name: kilo
-          context:
-            cluster: kilo
-            namespace: ${NAMESPACE}
-            user: kilo
-        current-context: kilo
-    EOF
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kilo
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/name: kilo
-    app.kubernetes.io/part-of: kilo
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: kilo
-      app.kubernetes.io/part-of: kilo
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: kilo
-        app.kubernetes.io/part-of: kilo
-    spec:
-      serviceAccountName: kilo
-      hostNetwork: true
-      containers:
-      - name: kilo
-        image: squat/kilo:0.5.0
-        args:
-        - --kubeconfig=/etc/kubernetes/kubeconfig
-        - --hostname=$(NODE_NAME)
-        - --cni=false
-        - --compatibility=cilium
-        - --local=false
-        - --encapsulate=crosssubnet
-        - --clean-up-interface=true
-        - --log-level=all
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        ports:
-        - containerPort: 1107
-          name: metrics
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: kilo-dir
-          mountPath: /var/lib/kilo
-        - name: kubeconfig
-          mountPath: /etc/kubernetes
-          readOnly: true
-        - name: lib-modules
-          mountPath: /lib/modules
-          readOnly: true
-        - name: xtables-lock
-          mountPath: /run/xtables.lock
-          readOnly: false
-      initContainers:
-      - name: generate-kubeconfig
-        image: squat/kilo:0.5.0
-        command:
-        - /bin/sh
-        args:
-        - /scripts/init.sh
-        imagePullPolicy: Always
-        volumeMounts:
-        - name: kubeconfig
-          mountPath: /etc/kubernetes
-        - name: scripts
-          mountPath: /scripts/
-          readOnly: true
-        - name: k3s-agent
-          mountPath: /var/lib/rancher/k3s/agent/
-          readOnly: true
-        env:
-        - name: NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-      volumes:
-      - name: kilo-dir
-        hostPath:
-          path: /var/lib/kilo
-      - name: kubeconfig
-        emptyDir: {}
-      - name: scripts
-        configMap:
-          name: kilo-scripts
-      - name: k3s-agent
-        hostPath:
-          path: /var/lib/rancher/k3s/agent
-      - name: lib-modules
-        hostPath:
-          path: /lib/modules
-      - name: xtables-lock
-        hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate

manifests/kilo-k3s-userspace-heterogeneous.yaml

@@ -300,7 +300,7 @@ spec:
       - name: boringtun
         image: leonnicolas/boringtun:cc19859
         args:
-        - --disable-drop-privileges=true
+        - --disable-drop-privileges
         - --foreground
         - kilo0
         securityContext:

manifests/kilo-k3s-userspace.yaml

@@ -167,7 +167,7 @@ spec:
       - name: boringtun
         image: leonnicolas/boringtun:cc19859
         args:
-        - --disable-drop-privileges=true
+        - --disable-drop-privileges
         - --foreground
         - kilo0
         securityContext:

manifests/kilo-kubeadm-userspace.yaml

@@ -104,7 +104,7 @@ spec:
         image: leonnicolas/boringtun:cc19859
         imagePullPolicy: IfNotPresent
         args:
-          - --disable-drop-privileges=true
+          - --disable-drop-privileges
           - --foreground
           - kilo0
         securityContext:

pkg/iptables/iptables.go

@@ -25,7 +25,6 @@ import (
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 const ipv6ModuleDisabledPath = "/sys/module/ipv6/parameters/disable"
@@ -221,7 +220,6 @@ type Controller struct {
 	errors       chan error
 	logger       log.Logger
 	resyncPeriod time.Duration
-	registerer   prometheus.Registerer
 
 	sync.Mutex
 	rules      []Rule
@@ -253,12 +251,6 @@ func WithClients(v4, v6 Client) ControllerOption {
 	}
 }
 
-func WithRegisterer(registerer prometheus.Registerer) ControllerOption {
-	return func(c *Controller) {
-		c.registerer = registerer
-	}
-}
-
 // New generates a new iptables rules controller.
 // If no options are given, IPv4 and IPv6 clients
 // will be instantiated using the regular iptables backend.
@@ -275,7 +267,7 @@ func New(opts ...ControllerOption) (*Controller, error) {
 		if err != nil {
 			return nil, fmt.Errorf("failed to create iptables IPv4 client: %v", err)
 		}
-		c.v4 = wrapWithMetrics(v4, "IPv4", c.registerer)
+		c.v4 = v4
 	}
 	if c.v6 == nil {
 		disabled, err := ipv6Disabled()
@@ -290,7 +282,7 @@ func New(opts ...ControllerOption) (*Controller, error) {
 			if err != nil {
 				return nil, fmt.Errorf("failed to create iptables IPv6 client: %v", err)
 			}
-			c.v6 = wrapWithMetrics(v6, "IPv6", c.registerer)
+			c.v6 = v6
 		}
 	}
 	return c, nil

pkg/iptables/metrics.go

@@ -1,115 +0,0 @@
-// Copyright 2022 the Kilo authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package iptables
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-type metricsClientWrapper struct {
-	client           Client
-	operationCounter *prometheus.CounterVec
-}
-
-func wrapWithMetrics(client Client, protocol string, registerer prometheus.Registerer) Client {
-	if registerer == nil {
-		return client
-	}
-
-	labelNames := []string{
-		"operation",
-		"table",
-		"chain",
-	}
-	counter := prometheus.NewCounterVec(prometheus.CounterOpts{
-		Name:        "kilo_iptables_operations_total",
-		Help:        "Number of iptables operations.",
-		ConstLabels: prometheus.Labels{"protocol": protocol},
-	}, labelNames)
-	registerer.MustRegister(counter)
-	return &metricsClientWrapper{client, counter}
-}
-
-func (m *metricsClientWrapper) AppendUnique(table string, chain string, rule ...string) error {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "AppendUnique",
-		"table":     table,
-		"chain":     chain,
-	}).Inc()
-	return m.client.AppendUnique(table, chain, rule...)
-}
-
-func (m *metricsClientWrapper) Delete(table string, chain string, rule ...string) error {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "Delete",
-		"table":     table,
-		"chain":     chain,
-	}).Inc()
-	return m.client.Delete(table, chain, rule...)
-}
-
-func (m *metricsClientWrapper) Exists(table string, chain string, rule ...string) (bool, error) {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "Exists",
-		"table":     table,
-		"chain":     chain,
-	}).Inc()
-	return m.client.Exists(table, chain, rule...)
-}
-
-func (m *metricsClientWrapper) List(table string, chain string) ([]string, error) {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "List",
-		"table":     table,
-		"chain":     chain,
-	}).Inc()
-	return m.client.List(table, chain)
-}
-
-func (m *metricsClientWrapper) ClearChain(table string, chain string) error {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "ClearChain",
-		"table":     table,
-		"chain":     chain,
-	}).Inc()
-	return m.client.ClearChain(table, chain)
-}
-
-func (m *metricsClientWrapper) DeleteChain(table string, chain string) error {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "DeleteChain",
-		"table":     table,
-		"chain":     chain,
-	}).Inc()
-	return m.client.DeleteChain(table, chain)
-}
-
-func (m *metricsClientWrapper) NewChain(table string, chain string) error {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "NewChain",
-		"table":     table,
-		"chain":     chain,
-	}).Inc()
-	return m.client.NewChain(table, chain)
-}
-
-func (m *metricsClientWrapper) ListChains(table string) ([]string, error) {
-	m.operationCounter.With(prometheus.Labels{
-		"operation": "ListChains",
-		"table":     table,
-		"chain":     "*",
-	}).Inc()
-	return m.client.ListChains(table)
-}

pkg/mesh/mesh.go

@@ -88,7 +88,7 @@ type Mesh struct {
 }
 
 // New returns a new Mesh instance.
-func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) {
+func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, logger log.Logger) (*Mesh, error) {
 	if err := os.MkdirAll(kiloPath, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create directory to store configuration: %v", err)
 	}
@@ -156,11 +156,11 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 		externalIP = publicIP
 	}
 	level.Debug(logger).Log("msg", fmt.Sprintf("using %s as the public IP address", publicIP.String()))
-	ipTables, err := iptables.New(iptables.WithRegisterer(registerer), iptables.WithLogger(log.With(logger, "component", "iptables")), iptables.WithResyncPeriod(resyncPeriod))
+	ipTables, err := iptables.New(iptables.WithLogger(log.With(logger, "component", "iptables")), iptables.WithResyncPeriod(resyncPeriod))
 	if err != nil {
 		return nil, fmt.Errorf("failed to IP tables controller: %v", err)
 	}
-	mesh := Mesh{
+	return &Mesh{
 		Backend:             backend,
 		cleanUpIface:        cleanUpIface,
 		cni:                 cni,
@@ -205,15 +205,7 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 			Help: "Number of reconciliation attempts.",
 		}),
 		logger: logger,
-	}
+	}, nil
-	registerer.MustRegister(
-		mesh.errorCounter,
-		mesh.leaderGuage,
-		mesh.nodesGuage,
-		mesh.peersGuage,
-		mesh.reconcileCounter,
-	)
-	return &mesh, nil
 }
 
 // Run starts the mesh.
@@ -524,9 +516,7 @@ func (m *Mesh) applyTopology() {
 				break
 			}
 		}
-
+		ipRules = append(ipRules, m.enc.Rules(cidrs)...)
-		ipRules = append(m.enc.Rules(cidrs), ipRules...)
-
 		// If we are handling local routes, ensure the local
 		// tunnel has an IP address.
 		if err := m.enc.Set(oneAddressCIDR(newAllocator(*nodes[m.hostname].Subnet).next().IP)); err != nil {
@@ -583,6 +573,18 @@ func (m *Mesh) applyTopology() {
 	}
 }
 
+// RegisterMetrics registers Prometheus metrics on the given Prometheus
+// registerer.
+func (m *Mesh) RegisterMetrics(r prometheus.Registerer) {
+	r.MustRegister(
+		m.errorCounter,
+		m.leaderGuage,
+		m.nodesGuage,
+		m.peersGuage,
+		m.reconcileCounter,
+	)
+}
+
 func (m *Mesh) cleanUp() {
 	if err := m.ipTables.CleanUp(); err != nil {
 		level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up IP tables: %v", err))

website/src/pages/index.js

@@ -12,7 +12,7 @@ const features = [
     imageUrl: 'https://kubernetes.io/images/nav_logo.svg',
     description: (
       <>
-        Kilo can be installed on any Kubernetes cluster, allowing nodes located in different clouds or in different countries to form a single cluster.
+        Kilo can be installed on any Kubernetes cluster, allowing nodes located in different clouds or in different coutries to form a single cluster.
       </>
     ),
     clip: true,

website/yarn.lock

@@ -2543,9 +2543,9 @@ browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.16.0, browserslist@^4
     node-releases "^1.1.71"
 
 buffer-from@^1.0.0:
-  version "1.1.2"
+  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
+  resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
-  integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
+  integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
 buffer-indexof@^1.0.0:
   version "1.1.1"
@@ -7752,9 +7752,9 @@ source-map-resolve@^0.5.0:
     urix "^0.1.0"
 
 source-map-support@~0.5.12, source-map-support@~0.5.19:
-  version "0.5.21"
+  version "0.5.19"
-  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
+  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61"
-  integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==
+  integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==
   dependencies:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
@@ -8060,9 +8060,9 @@ terser-webpack-plugin@^5.1.3:
     terser "^5.7.0"
 
 terser@^4.6.3:
-  version "4.8.1"
+  version "4.8.0"
-  resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.1.tgz#a00e5634562de2239fd404c649051bf6fc21144f"
+  resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.0.tgz#63056343d7c70bb29f3af665865a46fe03a0df17"
-  integrity sha512-4GnLC0x667eJG0ewJTa6z/yXrbLGv80D9Ru6HIpCQmO+Q4PfEtBFi0ObSckqwL6VyQv/7ENJieXHo2ANmdQwgw==
+  integrity sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==
   dependencies:
     commander "^2.20.0"
     source-map "~0.6.1"