2019-01-18 01:50:10 +00:00
|
|
|
/*
|
|
|
|
Copyright 2015 The Kubernetes Authors.
|
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package flowcontrol
|
|
|
|
|
|
|
|
import (
|
2022-04-23 09:01:19 +00:00
|
|
|
"math/rand"
|
2019-01-18 01:50:10 +00:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2022-04-23 09:01:19 +00:00
|
|
|
"k8s.io/utils/clock"
|
|
|
|
testingclock "k8s.io/utils/clock/testing"
|
2019-05-03 10:50:21 +00:00
|
|
|
"k8s.io/utils/integer"
|
2019-01-18 01:50:10 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type backoffEntry struct {
|
|
|
|
backoff time.Duration
|
|
|
|
lastUpdate time.Time
|
|
|
|
}
|
|
|
|
|
|
|
|
type Backoff struct {
|
2021-05-15 10:08:31 +00:00
|
|
|
sync.RWMutex
|
2019-01-18 01:50:10 +00:00
|
|
|
Clock clock.Clock
|
|
|
|
defaultDuration time.Duration
|
|
|
|
maxDuration time.Duration
|
|
|
|
perItemBackoff map[string]*backoffEntry
|
2022-04-23 09:01:19 +00:00
|
|
|
rand *rand.Rand
|
|
|
|
|
|
|
|
// maxJitterFactor adds jitter to the exponentially backed off delay.
|
|
|
|
// if maxJitterFactor is zero, no jitter is added to the delay in
|
|
|
|
// order to maintain current behavior.
|
|
|
|
maxJitterFactor float64
|
2019-01-18 01:50:10 +00:00
|
|
|
}
|
|
|
|
|
2022-04-23 09:01:19 +00:00
|
|
|
func NewFakeBackOff(initial, max time.Duration, tc *testingclock.FakeClock) *Backoff {
|
|
|
|
return newBackoff(tc, initial, max, 0.0)
|
2019-01-18 01:50:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func NewBackOff(initial, max time.Duration) *Backoff {
|
2022-04-23 09:01:19 +00:00
|
|
|
return NewBackOffWithJitter(initial, max, 0.0)
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewFakeBackOffWithJitter(initial, max time.Duration, tc *testingclock.FakeClock, maxJitterFactor float64) *Backoff {
|
|
|
|
return newBackoff(tc, initial, max, maxJitterFactor)
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewBackOffWithJitter(initial, max time.Duration, maxJitterFactor float64) *Backoff {
|
|
|
|
clock := clock.RealClock{}
|
|
|
|
return newBackoff(clock, initial, max, maxJitterFactor)
|
|
|
|
}
|
|
|
|
|
|
|
|
func newBackoff(clock clock.Clock, initial, max time.Duration, maxJitterFactor float64) *Backoff {
|
|
|
|
var random *rand.Rand
|
|
|
|
if maxJitterFactor > 0 {
|
|
|
|
random = rand.New(rand.NewSource(clock.Now().UnixNano()))
|
|
|
|
}
|
2019-01-18 01:50:10 +00:00
|
|
|
return &Backoff{
|
|
|
|
perItemBackoff: map[string]*backoffEntry{},
|
2022-04-23 09:01:19 +00:00
|
|
|
Clock: clock,
|
2019-01-18 01:50:10 +00:00
|
|
|
defaultDuration: initial,
|
|
|
|
maxDuration: max,
|
2022-04-23 09:01:19 +00:00
|
|
|
maxJitterFactor: maxJitterFactor,
|
|
|
|
rand: random,
|
2019-01-18 01:50:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the current backoff Duration
|
|
|
|
func (p *Backoff) Get(id string) time.Duration {
|
2021-05-15 10:08:31 +00:00
|
|
|
p.RLock()
|
|
|
|
defer p.RUnlock()
|
2019-01-18 01:50:10 +00:00
|
|
|
var delay time.Duration
|
|
|
|
entry, ok := p.perItemBackoff[id]
|
|
|
|
if ok {
|
|
|
|
delay = entry.backoff
|
|
|
|
}
|
|
|
|
return delay
|
|
|
|
}
|
|
|
|
|
|
|
|
// move backoff to the next mark, capping at maxDuration
|
|
|
|
func (p *Backoff) Next(id string, eventTime time.Time) {
|
|
|
|
p.Lock()
|
|
|
|
defer p.Unlock()
|
|
|
|
entry, ok := p.perItemBackoff[id]
|
|
|
|
if !ok || hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
|
|
|
|
entry = p.initEntryUnsafe(id)
|
2022-04-23 09:01:19 +00:00
|
|
|
entry.backoff += p.jitter(entry.backoff)
|
2019-01-18 01:50:10 +00:00
|
|
|
} else {
|
2022-04-23 09:01:19 +00:00
|
|
|
delay := entry.backoff * 2 // exponential
|
|
|
|
delay += p.jitter(entry.backoff) // add some jitter to the delay
|
2019-01-18 01:50:10 +00:00
|
|
|
entry.backoff = time.Duration(integer.Int64Min(int64(delay), int64(p.maxDuration)))
|
|
|
|
}
|
|
|
|
entry.lastUpdate = p.Clock.Now()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reset forces clearing of all backoff data for a given key.
|
|
|
|
func (p *Backoff) Reset(id string) {
|
|
|
|
p.Lock()
|
|
|
|
defer p.Unlock()
|
|
|
|
delete(p.perItemBackoff, id)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Returns True if the elapsed time since eventTime is smaller than the current backoff window
|
|
|
|
func (p *Backoff) IsInBackOffSince(id string, eventTime time.Time) bool {
|
2021-05-15 10:08:31 +00:00
|
|
|
p.RLock()
|
|
|
|
defer p.RUnlock()
|
2019-01-18 01:50:10 +00:00
|
|
|
entry, ok := p.perItemBackoff[id]
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
|
|
|
|
return false
|
|
|
|
}
|
2021-05-15 10:08:31 +00:00
|
|
|
return p.Clock.Since(eventTime) < entry.backoff
|
2019-01-18 01:50:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Returns True if time since lastupdate is less than the current backoff window.
|
|
|
|
func (p *Backoff) IsInBackOffSinceUpdate(id string, eventTime time.Time) bool {
|
2021-05-15 10:08:31 +00:00
|
|
|
p.RLock()
|
|
|
|
defer p.RUnlock()
|
2019-01-18 01:50:10 +00:00
|
|
|
entry, ok := p.perItemBackoff[id]
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return eventTime.Sub(entry.lastUpdate) < entry.backoff
|
|
|
|
}
|
|
|
|
|
|
|
|
// Garbage collect records that have aged past maxDuration. Backoff users are expected
|
|
|
|
// to invoke this periodically.
|
|
|
|
func (p *Backoff) GC() {
|
|
|
|
p.Lock()
|
|
|
|
defer p.Unlock()
|
|
|
|
now := p.Clock.Now()
|
|
|
|
for id, entry := range p.perItemBackoff {
|
|
|
|
if now.Sub(entry.lastUpdate) > p.maxDuration*2 {
|
|
|
|
// GC when entry has not been updated for 2*maxDuration
|
|
|
|
delete(p.perItemBackoff, id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *Backoff) DeleteEntry(id string) {
|
|
|
|
p.Lock()
|
|
|
|
defer p.Unlock()
|
|
|
|
delete(p.perItemBackoff, id)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Take a lock on *Backoff, before calling initEntryUnsafe
|
|
|
|
func (p *Backoff) initEntryUnsafe(id string) *backoffEntry {
|
|
|
|
entry := &backoffEntry{backoff: p.defaultDuration}
|
|
|
|
p.perItemBackoff[id] = entry
|
|
|
|
return entry
|
|
|
|
}
|
|
|
|
|
2022-04-23 09:01:19 +00:00
|
|
|
func (p *Backoff) jitter(delay time.Duration) time.Duration {
|
|
|
|
if p.rand == nil {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
return time.Duration(p.rand.Float64() * p.maxJitterFactor * float64(delay))
|
|
|
|
}
|
|
|
|
|
2019-01-18 01:50:10 +00:00
|
|
|
// After 2*maxDuration we restart the backoff factor to the beginning
|
|
|
|
func hasExpired(eventTime time.Time, lastUpdate time.Time, maxDuration time.Duration) bool {
|
|
|
|
return eventTime.Sub(lastUpdate) > maxDuration*2 // consider stable if it's ok for twice the maxDuration
|
|
|
|
}
|