From 4be792ea543a9c2656574ec060b335c587244a3d Mon Sep 17 00:00:00 2001 From: Antoine Date: Fri, 20 May 2022 02:13:07 +0200 Subject: [PATCH] feat: cilium add-mode support (#312) * feat: cilium add-mode support when cni management by kilo is disable, we can use existing cluster's cni setup thanks to add-on mode https://kilo.squat.ai/docs/introduction#add-on-mode * feat: manifest example for cilium addon mode * fix: apply comment from PR review * fix: add mutex to interface retrieval into flannel addon mode --- cmd/kg/main.go | 2 + manifests/kilo-kubeadm-cilium.yaml | 142 +++++++++++++++++++++++++++++ pkg/encapsulation/cilium.go | 111 ++++++++++++++++++++++ pkg/encapsulation/flannel.go | 2 + 4 files changed, 257 insertions(+) create mode 100644 manifests/kilo-kubeadm-cilium.yaml create mode 100644 pkg/encapsulation/cilium.go diff --git a/cmd/kg/main.go b/cmd/kg/main.go index c1b4870..1834653 100644 --- a/cmd/kg/main.go +++ b/cmd/kg/main.go @@ -213,6 +213,8 @@ func runRoot(_ *cobra.Command, _ []string) error { switch compatibility { case "flannel": enc = encapsulation.NewFlannel(e) + case "cilium": + enc = encapsulation.NewCilium(e) default: enc = encapsulation.NewIPIP(e) } diff --git a/manifests/kilo-kubeadm-cilium.yaml b/manifests/kilo-kubeadm-cilium.yaml new file mode 100644 index 0000000..56cf4f5 --- /dev/null +++ b/manifests/kilo-kubeadm-cilium.yaml @@ -0,0 +1,142 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kilo + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kilo +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - patch + - watch +- apiGroups: + - kilo.squat.ai + resources: + - peers + verbs: + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kilo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kilo +subjects: + - kind: ServiceAccount + name: kilo + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kilo + namespace: kube-system + labels: + app.kubernetes.io/name: kilo + app.kubernetes.io/part-of: kilo +spec: + selector: + matchLabels: + app.kubernetes.io/name: kilo + app.kubernetes.io/part-of: kilo + template: + metadata: + labels: + app.kubernetes.io/name: kilo + app.kubernetes.io/part-of: kilo + spec: + serviceAccountName: kilo + hostNetwork: true + containers: + - name: kilo + image: squat/kilo:0.5.0 + args: + - --kubeconfig=/etc/kubernetes/kubeconfig + - --hostname=$(NODE_NAME) + - --cni=false + - --compatibility=cilium + - --local=false + # additional and also optional flag + - --encapsulate=crosssubnet + - --clean-up-interface=true + - --subnet=172.31.254.0/24 + - --log-level=all + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 1107 + name: metrics + securityContext: + privileged: true + volumeMounts: + - name: kilo-dir + mountPath: /var/lib/kilo + + # with kube-proxy configmap + # - name: kubeconfig + # mountPath: /etc/kubernetes + # readOnly: true + + # without kube-proxy host kubeconfig binding + - name: kubeconfig + mount_path: /etc/kubernetes/kubeconfig + sub_path: admin.conf + read_only: true + + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: kilo-dir + hostPath: + path: /var/lib/kilo + + # with kube-proxy configmap + # - name: kubeconfig + # configMap: + # name: kube-proxy + # items: + # - key: kubeconfig.conf + # path: kubeconfig + + # without kube-proxy host kubeconfig binding + - name: kubeconfig + host_path: + path: /etc/kubernetes + + - name: lib-modules + hostPath: + path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate diff --git a/pkg/encapsulation/cilium.go b/pkg/encapsulation/cilium.go new file mode 100644 index 0000000..9e34292 --- /dev/null +++ b/pkg/encapsulation/cilium.go @@ -0,0 +1,111 @@ +// Copyright 2019 the Kilo authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encapsulation + +import ( + "fmt" + "net" + "sync" + + "github.com/vishvananda/netlink" + + "github.com/squat/kilo/pkg/iptables" +) + +const ciliumDeviceName = "cilium_host" + +type cilium struct { + iface int + strategy Strategy + ch chan netlink.LinkUpdate + done chan struct{} + // mu guards updates to the iface field. + mu sync.Mutex +} + +// NewCilium returns an encapsulator that uses Cilium. +func NewCilium(strategy Strategy) Encapsulator { + return &cilium{ + ch: make(chan netlink.LinkUpdate), + done: make(chan struct{}), + strategy: strategy, + } +} + +// CleanUp close done channel +func (f *cilium) CleanUp() error { + close(f.done) + return nil +} + +// Gw returns the correct gateway IP associated with the given node. +func (f *cilium) Gw(_, _ net.IP, subnet *net.IPNet) net.IP { + return subnet.IP +} + +// Index returns the index of the Cilium interface. +func (f *cilium) Index() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.iface +} + +// Init finds the Cilium interface index. +func (f *cilium) Init(_ int) error { + if err := netlink.LinkSubscribe(f.ch, f.done); err != nil { + return fmt.Errorf("failed to subscribe to updates to %s: %v", ciliumDeviceName, err) + } + go func() { + var lu netlink.LinkUpdate + for { + select { + case lu = <-f.ch: + if lu.Attrs().Name == ciliumDeviceName { + f.mu.Lock() + f.iface = lu.Attrs().Index + f.mu.Unlock() + } + case <-f.done: + return + } + } + }() + i, err := netlink.LinkByName(ciliumDeviceName) + if _, ok := err.(netlink.LinkNotFoundError); ok { + return nil + } + if err != nil { + return fmt.Errorf("failed to query for Cilium interface: %v", err) + } + f.mu.Lock() + f.iface = i.Attrs().Index + f.mu.Unlock() + return nil +} + +// Rules is a no-op. +func (f *cilium) Rules(_ []*net.IPNet) []iptables.Rule { + return nil +} + +// Set is a no-op. +func (f *cilium) Set(_ *net.IPNet) error { + return nil +} + +// Strategy returns the configured strategy for encapsulation. +func (f *cilium) Strategy() Strategy { + return f.strategy +} diff --git a/pkg/encapsulation/flannel.go b/pkg/encapsulation/flannel.go index 6a2ba26..e08af61 100644 --- a/pkg/encapsulation/flannel.go +++ b/pkg/encapsulation/flannel.go @@ -56,6 +56,8 @@ func (f *flannel) Gw(_, _ net.IP, subnet *net.IPNet) net.IP { // Index returns the index of the Flannel interface. func (f *flannel) Index() int { + f.mu.Lock() + defer f.mu.Unlock() return f.iface }