diff --git a/cmd/kg/main.go b/cmd/kg/main.go index 1834653..11f20b9 100644 --- a/cmd/kg/main.go +++ b/cmd/kg/main.go @@ -245,13 +245,11 @@ func runRoot(_ *cobra.Command, _ []string) error { if port < 1 || port > 1<<16-1 { return fmt.Errorf("invalid port: port mus be in range [%d:%d], but got %d", 1, 1<<16-1, port) } - m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, log.With(logger, "component", "kilo")) + m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, log.With(logger, "component", "kilo"), registry) if err != nil { return fmt.Errorf("failed to create Kilo mesh: %v", err) } - m.RegisterMetrics(registry) - var g run.Group { h := internalserver.NewHandler( diff --git a/pkg/iptables/iptables.go b/pkg/iptables/iptables.go index 8290676..4cad47c 100644 --- a/pkg/iptables/iptables.go +++ b/pkg/iptables/iptables.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/go-iptables/iptables" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" ) const ipv6ModuleDisabledPath = "/sys/module/ipv6/parameters/disable" @@ -220,6 +221,7 @@ type Controller struct { errors chan error logger log.Logger resyncPeriod time.Duration + registerer prometheus.Registerer sync.Mutex rules []Rule @@ -251,6 +253,12 @@ func WithClients(v4, v6 Client) ControllerOption { } } +func WithRegisterer(registerer prometheus.Registerer) ControllerOption { + return func(c *Controller) { + c.registerer = registerer + } +} + // New generates a new iptables rules controller. // If no options are given, IPv4 and IPv6 clients // will be instantiated using the regular iptables backend. @@ -267,7 +275,7 @@ func New(opts ...ControllerOption) (*Controller, error) { if err != nil { return nil, fmt.Errorf("failed to create iptables IPv4 client: %v", err) } - c.v4 = v4 + c.v4 = wrapWithMetrics(v4, "IPv4", c.registerer) } if c.v6 == nil { disabled, err := ipv6Disabled() @@ -282,7 +290,7 @@ func New(opts ...ControllerOption) (*Controller, error) { if err != nil { return nil, fmt.Errorf("failed to create iptables IPv6 client: %v", err) } - c.v6 = v6 + c.v6 = wrapWithMetrics(v6, "IPv6", c.registerer) } } return c, nil diff --git a/pkg/iptables/metrics.go b/pkg/iptables/metrics.go new file mode 100644 index 0000000..b262937 --- /dev/null +++ b/pkg/iptables/metrics.go @@ -0,0 +1,115 @@ +// Copyright 2022 the Kilo authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iptables + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +type metricsClientWrapper struct { + client Client + operationCounter *prometheus.CounterVec +} + +func wrapWithMetrics(client Client, protocol string, registerer prometheus.Registerer) Client { + if registerer == nil { + return client + } + + labelNames := []string{ + "operation", + "table", + "chain", + } + counter := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "kilo_iptables_operations_total", + Help: "Number of iptables operations.", + ConstLabels: prometheus.Labels{"protocol": protocol}, + }, labelNames) + registerer.MustRegister(counter) + return &metricsClientWrapper{client, counter} +} + +func (m *metricsClientWrapper) AppendUnique(table string, chain string, rule ...string) error { + m.operationCounter.With(prometheus.Labels{ + "operation": "AppendUnique", + "table": table, + "chain": chain, + }).Inc() + return m.client.AppendUnique(table, chain, rule...) +} + +func (m *metricsClientWrapper) Delete(table string, chain string, rule ...string) error { + m.operationCounter.With(prometheus.Labels{ + "operation": "Delete", + "table": table, + "chain": chain, + }).Inc() + return m.client.Delete(table, chain, rule...) +} + +func (m *metricsClientWrapper) Exists(table string, chain string, rule ...string) (bool, error) { + m.operationCounter.With(prometheus.Labels{ + "operation": "Exists", + "table": table, + "chain": chain, + }).Inc() + return m.client.Exists(table, chain, rule...) +} + +func (m *metricsClientWrapper) List(table string, chain string) ([]string, error) { + m.operationCounter.With(prometheus.Labels{ + "operation": "List", + "table": table, + "chain": chain, + }).Inc() + return m.client.List(table, chain) +} + +func (m *metricsClientWrapper) ClearChain(table string, chain string) error { + m.operationCounter.With(prometheus.Labels{ + "operation": "ClearChain", + "table": table, + "chain": chain, + }).Inc() + return m.client.ClearChain(table, chain) +} + +func (m *metricsClientWrapper) DeleteChain(table string, chain string) error { + m.operationCounter.With(prometheus.Labels{ + "operation": "DeleteChain", + "table": table, + "chain": chain, + }).Inc() + return m.client.DeleteChain(table, chain) +} + +func (m *metricsClientWrapper) NewChain(table string, chain string) error { + m.operationCounter.With(prometheus.Labels{ + "operation": "NewChain", + "table": table, + "chain": chain, + }).Inc() + return m.client.NewChain(table, chain) +} + +func (m *metricsClientWrapper) ListChains(table string) ([]string, error) { + m.operationCounter.With(prometheus.Labels{ + "operation": "ListChains", + "table": table, + "chain": "*", + }).Inc() + return m.client.ListChains(table) +} diff --git a/pkg/mesh/mesh.go b/pkg/mesh/mesh.go index 22c54e1..f1091dc 100644 --- a/pkg/mesh/mesh.go +++ b/pkg/mesh/mesh.go @@ -88,7 +88,7 @@ type Mesh struct { } // New returns a new Mesh instance. -func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, logger log.Logger) (*Mesh, error) { +func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) { if err := os.MkdirAll(kiloPath, 0700); err != nil { return nil, fmt.Errorf("failed to create directory to store configuration: %v", err) } @@ -156,11 +156,11 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit externalIP = publicIP } level.Debug(logger).Log("msg", fmt.Sprintf("using %s as the public IP address", publicIP.String())) - ipTables, err := iptables.New(iptables.WithLogger(log.With(logger, "component", "iptables")), iptables.WithResyncPeriod(resyncPeriod)) + ipTables, err := iptables.New(iptables.WithRegisterer(registerer), iptables.WithLogger(log.With(logger, "component", "iptables")), iptables.WithResyncPeriod(resyncPeriod)) if err != nil { return nil, fmt.Errorf("failed to IP tables controller: %v", err) } - return &Mesh{ + mesh := Mesh{ Backend: backend, cleanUpIface: cleanUpIface, cni: cni, @@ -205,7 +205,15 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit Help: "Number of reconciliation attempts.", }), logger: logger, - }, nil + } + registerer.MustRegister( + mesh.errorCounter, + mesh.leaderGuage, + mesh.nodesGuage, + mesh.peersGuage, + mesh.reconcileCounter, + ) + return &mesh, nil } // Run starts the mesh. @@ -575,18 +583,6 @@ func (m *Mesh) applyTopology() { } } -// RegisterMetrics registers Prometheus metrics on the given Prometheus -// registerer. -func (m *Mesh) RegisterMetrics(r prometheus.Registerer) { - r.MustRegister( - m.errorCounter, - m.leaderGuage, - m.nodesGuage, - m.peersGuage, - m.reconcileCounter, - ) -} - func (m *Mesh) cleanUp() { if err := m.ipTables.CleanUp(); err != nil { level.Error(m.logger).Log("error", fmt.Sprintf("failed to clean up IP tables: %v", err))