vrr/Pulse
2
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-07-09 16:00:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Fold pulse-agent module init retry into one generic helper

Browse source 
initDockerWithRetry and initKubernetesWithRetry were the same exponential
backoff loop (5s doubling to a 5m cap, cancel-aware wait, structured
retry logging) duplicated per module; dupl flagged the pair. Extract
initModuleWithRetry[T] and keep the two as thin wrappers so the
test-overridable newDockerAgent/newKubeAgent vars and call sites stay
unchanged. Per-module log strings are preserved verbatim.

Existing cmd/pulse-agent retry tests (success, cancel-before-connect,
cancel-during-wait, both modules) pass unchanged.
This commit is contained in:
rcourtman 2026-06-10 09:18:30 +01:00
parent 74417de150
commit e12387489e
1 changed files with 28 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

87
cmd/pulse-agent/main.go
View file

@ -1115,46 +1115,49 @@ func resolveTokenInternal(tokenFlag, tokenFileFlag, envToken string, readFile fu
return ""
}
// initDockerWithRetry attempts to initialize the Docker / Podman collection module with exponential backoff.
// It returns the module when Docker / Podman becomes available, or nil if the context is cancelled.
// initModuleWithRetry attempts to initialize an agent module with exponential
// backoff. It returns the module when its backend becomes available, or a zero
// value if the context is cancelled. component and displayName feed the
// structured log fields; unavailableMsg is the per-module retry log line.
// Retry intervals: 5s, 10s, 20s, 40s, 80s, 160s, then cap at 5 minutes.
func initDockerWithRetry(ctx context.Context, cfg dockeragent.Config, logger *zerolog.Logger) RunnableCloser {
func initModuleWithRetry[T any](ctx context.Context, logger *zerolog.Logger, component, displayName, unavailableMsg string, connect func() (T, error)) T {
const multiplier = 2.0
var zero T
delay := retryInitialDelay
attempt := 0
for {
if err := ctx.Err(); err != nil {
logger.Info().Msg("Docker retry cancelled, context done")
return nil
logger.Info().Msg(displayName + " retry cancelled, context done")
return zero
}
agent, err := newDockerAgent(cfg)
module, err := connect()
if err == nil {
logger.Info().
Str("component", "docker_agent").
Str("component", component).
Str("action", "retry_connect_succeeded").
Int("attempts", attempt+1).
Msg("Successfully connected to Docker after retry")
return agent
Msg("Successfully connected to " + displayName + " after retry")
return module
}
attempt++
retryLogEvent(logger, attempt).
Err(err).
Str("component", "docker_agent").
Str("component", component).
Str("action", "retry_connect_failed").
Int("attempt", attempt).
Str("next_retry", delay.String()).
Msg("Docker not available, will retry")
Msg(unavailableMsg)
if !waitForRetryDelay(ctx, delay) {
logger.Info().
Str("component", "docker_agent").
Str("component", component).
Str("action", "retry_cancelled").
Msg("Docker retry cancelled, context done")
return nil
Msg(displayName + " retry cancelled, context done")
return zero
}
// Calculate next delay with exponential backoff, capped at retryMaxDelay
@ -1165,54 +1168,20 @@ func initDockerWithRetry(ctx context.Context, cfg dockeragent.Config, logger *ze
}
}
// initDockerWithRetry attempts to initialize the Docker / Podman collection module with exponential backoff.
// It returns the module when Docker / Podman becomes available, or nil if the context is cancelled.
func initDockerWithRetry(ctx context.Context, cfg dockeragent.Config, logger *zerolog.Logger) RunnableCloser {
return initModuleWithRetry(ctx, logger, "docker_agent", "Docker", "Docker not available, will retry", func() (RunnableCloser, error) {
return newDockerAgent(cfg)
})
}
// initKubernetesWithRetry attempts to initialize the Kubernetes agent with exponential backoff.
// It returns the agent when Kubernetes becomes available, or nil if the context is cancelled.
// Retry intervals: 5s, 10s, 20s, 40s, 80s, 160s, then cap at 5 minutes.
func initKubernetesWithRetry(ctx context.Context, cfg kubernetesagent.Config, logger *zerolog.Logger) Runnable {
const multiplier = 2.0
delay := retryInitialDelay
attempt := 0
for {
if err := ctx.Err(); err != nil {
logger.Info().Msg("Kubernetes retry cancelled, context done")
return nil
}
agent, err := newKubeAgent(cfg)
if err == nil {
logger.Info().
Str("component", "kubernetes_agent").
Str("action", "retry_connect_succeeded").
Int("attempts", attempt+1).
Msg("Successfully connected to Kubernetes after retry")
return agent
}
attempt++
retryLogEvent(logger, attempt).
Err(err).
Str("component", "kubernetes_agent").
Str("action", "retry_connect_failed").
Int("attempt", attempt).
Str("next_retry", delay.String()).
Msg("Kubernetes still not available, will retry")
if !waitForRetryDelay(ctx, delay) {
logger.Info().
Str("component", "kubernetes_agent").
Str("action", "retry_cancelled").
Msg("Kubernetes retry cancelled, context done")
return nil
}
// Calculate next delay with exponential backoff, capped at retryMaxDelay
delay = time.Duration(float64(delay) * multiplier)
if delay > retryMaxDelay {
delay = retryMaxDelay
}
}
return initModuleWithRetry(ctx, logger, "kubernetes_agent", "Kubernetes", "Kubernetes still not available, will retry", func() (Runnable, error) {
return newKubeAgent(cfg)
})
}
func waitForRetryDelay(ctx context.Context, delay time.Duration) bool {