diff --git a/Makefile b/Makefile index 4a7647f..a97faea 100644 --- a/Makefile +++ b/Makefile @@ -243,6 +243,7 @@ website/docs/README.md: README.md cp -r docs/graphs website/static/img/ sed -i 's/\.\/docs\///g' $@ find $(@D) -type f -name '*.md' | xargs -I{} sed -i 's/\.\/\(.\+\.svg\)/\/img\/\1/g' {} + find $(@D) -type f -name '*.md' | xargs -I{} sed -i 's/\.\/\(.\+\.png\)/\/img\/\1/g' {} sed -i 's/graphs\//\/img\/graphs\//g' $@ # The next line is a workaround until mdx, docusaurus' markdown parser, can parse links with preceding brackets. sed -i 's/\[\]\(\[.*\](.*)\)/\[\]\1/g' website/docs/api.md diff --git a/docs/graphs/kilo.png b/docs/graphs/kilo.png new file mode 100644 index 0000000..6e1c3de Binary files /dev/null and b/docs/graphs/kilo.png differ diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..d8d6c4f --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,100 @@ +# Monitoring + +The following assumes that you have applied the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) monitoring stack onto your cluster. + +## Kilo + +Monitor the Kilo daemon set with: +```shell +kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/podmonitor.yaml +``` + +## WireGuard + +Monitor the WireGuard interfaces with: +```shell +kubectl create ns kilo +kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/wg-exporter.yaml +``` + +The manifest will deploy [Prometheus WireGuard Exporter](https://github.com/MindFlavor/prometheus_wireguard_exporter) as a daemon set and a [podmonitor](https://docs.openshift.com/container-platform/4.8/rest_api/monitoring_apis/podmonitor-monitoring-coreos-com-v1.html). + +By default kube-prometheus will only monitor the default, kube-system and monitoring namespaces. +In order to allow prometheus-k8s to monitor the kilo namespace, apply the Role and RoleBinding with: +```shell +kubectl apply -f kubectl apply -f https://raw.githubusercontent.com/squat/kilo/main/manifests/wg-exporter-role-kube-prometheus.yaml +``` + +## Metrics + +### Kilo + +Kilo exports some standard metrics with the Prometheus GoCollector and ProcessCollector. +It also exposes some Kilo specific metrics. + +``` +# HELP kilo_errors_total Number of errors that occurred while administering the mesh. +# TYPE kilo_errors_total counter + +# HELP kilo_leader Leadership status of the node. +# TYPE kilo_leader gauge + +# HELP kilo_nodes Number of nodes in the mesh. +# TYPE kilo_nodes gauge + +# HELP kilo_peers Number of peers in the mesh. +# TYPE kilo_peers gauge + +# HELP kilo_reconciles_total Number of reconciliation attempts. +# TYPE kilo_reconciles_total counter +``` + +### WireGuard + +The [Prometheus WireGuard Exporter](https://github.com/MindFlavor/prometheus_wireguard_exporter) exports the following metrics: + +``` +# HELP wireguard_sent_bytes_total Bytes sent to the peer +# TYPE wireguard_sent_bytes_total counter + +# HELP wireguard_received_bytes_total Bytes received from the peer +# TYPE wireguard_received_bytes_total counter + +# HELP wireguard_latest_handshake_seconds Seconds from the last handshake +# TYPE wireguard_latest_handshake_seconds gauge +``` + +## Display some Metrics + +If your laptop is a Kilo peer of the cluster you can navigate you browser directly to the service IP of prometheus-k8s. +Otherwise use `port-forward`: +```shell +kubectl -n monitoring port-forward svc/prometheus-k8s 9090 +``` +and navigate your browser to `localhost:9090`. +Check if you can see the podmonitor of Kilo and the WireGuard Exporter under **Status** -> **Targets** in the web frontend. + +If you don't see them, check the logs of the `prometheus-k8s` pods, maybe they don't have the permission to get the pods in their namespaces. +In this case, you need to apply the Role and RoleBinding from above. + +Navigate to **Graph** and try to execute a simple query, eg. type `kilo_nodes` and klick execute. +You should see some data. + +## Using Grafana + +Let't navigate to the Grafana dashboard. +Again, if your laptop is not a Kilo peer, use `port-forward`: +```shell +kubectl -n monitoring port-forward svc/grafana 3000 +``` + +Now navigate your browser to `localhost:3000`. +The default user and password is `admin` `admin`. + +There is an example configuration for a dashboard [here](https://raw.githubusercontent.com/squat/kilo/main/docs/grafana/kilo.json). +You can import this dashboard if you hit **+** -> **Import** on the Grafana dashboard. + +The dashboard looks like this: + + + diff --git a/manifests/wg-exporter-role-kube-prometheus.yaml b/manifests/wg-exporter-role-kube-prometheus.yaml new file mode 100644 index 0000000..891d01c --- /dev/null +++ b/manifests/wg-exporter-role-kube-prometheus.yaml @@ -0,0 +1,56 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: prometheus + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 2.26.0 + name: prometheus-k8s + namespace: kilo +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: prometheus + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 2.26.0 + name: prometheus-k8s + namespace: kilo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-k8s +subjects: +- kind: ServiceAccount + name: prometheus-k8s + namespace: monitoring diff --git a/manifests/wg-exporter.yaml b/manifests/wg-exporter.yaml new file mode 100644 index 0000000..296d906 --- /dev/null +++ b/manifests/wg-exporter.yaml @@ -0,0 +1,67 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + app.kubernetes.io/name: wg-exporter + app.kubernetes.io/part-of: kilo + name: wg-exporter + namespace: kilo +spec: + namespaceSelector: + matchNames: + - kilo + podMetricsEndpoints: + - interval: 15s + port: metrics + path: /metrics + selector: + matchLabels: + app.kubernetes.io/part-of: kilo + app.kubernetes.io/name: wg-exporter +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: wg-exporter + app.kubernetes.io/part-of: kilo + name: wg-exporter + namespace: kilo +spec: + selector: + matchLabels: + app.kubernetes.io/name: wg-exporter + app.kubernetes.io/part-of: kilo + template: + metadata: + labels: + app.kubernetes.io/name: wg-exporter + app.kubernetes.io/part-of: kilo + spec: + containers: + - args: + - -a + - -i=kilo0 + - -p=9586 + image: mindflavor/prometheus-wireguard-exporter + name: wg-exporter + ports: + - containerPort: 9586 + name: metrics + protocol: TCP + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: wireguard + mountPath: /var/run/wireguard + volumes: + - name: wireguard + hostPath: + path: /var/run/wireguard + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists diff --git a/website/docs/monitoring b/website/docs/monitoring new file mode 100644 index 0000000..c83eac9 --- /dev/null +++ b/website/docs/monitoring @@ -0,0 +1,5 @@ +--- +id: monitoring +title: Monitoring +hide_title: true +--- diff --git a/website/sidebars.js b/website/sidebars.js index d977bc1..90c2870 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -7,7 +7,7 @@ module.exports = { { type: 'category', label: 'Guides', - items: ['topology', 'vpn', 'vpn-server', 'multi-cluster-services', 'network-policies', 'userspace-wireguard', 'peer-validation'], + items: ['topology', 'vpn', 'vpn-server', 'multi-cluster-services', 'network-policies', 'userspace-wireguard', 'peer-validation', 'monitoring'], }, { type: 'category',