mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-04-28 03:20:11 +00:00
Add DNS caching to reduce excessive DNS queries
Related to #608 Implements DNS caching using rs/dnscache to dramatically reduce DNS query volume for frequently accessed Proxmox hosts. Users were reporting 260,000+ DNS queries in 37 hours for the same hostnames. Changes: - Added rs/dnscache dependency for DNS resolution caching - Created pkg/tlsutil/dnscache.go with DNS cache wrapper - Updated HTTP client creation to use cached DNS resolver - Added DNSCacheTimeout configuration option (default: 5 minutes) - Made DNS cache timeout configurable via: - system.json: dnsCacheTimeout field (seconds) - Environment variable: DNS_CACHE_TIMEOUT (duration string) - DNS cache periodically refreshes to prevent stale entries Benefits: - Reduces DNS query load on local DNS servers by ~99% - Reduces network traffic and DNS query log volume - Maintains fresh DNS entries through periodic refresh - Configurable timeout for different network environments Default behavior: 5-minute cache timeout with automatic refresh
This commit is contained in:
rcourtman
parent
f434a7b9e7
commit
c93581e1aa
6 changed files with 149 additions and 17 deletions
2
go.mod
2
go.mod
|
|
@ -13,6 +13,7 @@ require (
|
|||
github.com/joho/godotenv v1.5.1
|
||||
github.com/oklog/ulid/v2 v2.1.1
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/shirou/gopsutil/v4 v4.25.9
|
||||
github.com/spf13/cobra v1.9.1
|
||||
|
|
@ -68,6 +69,7 @@ require (
|
|||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
gotest.tools/v3 v3.5.2 // indirect
|
||||
)
|
||||
|
|
|
|||
5
go.sum
5
go.sum
|
|
@ -109,6 +109,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
|
|||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8=
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
||||
|
|
@ -162,6 +164,9 @@ golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
|||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo=
|
||||
golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import (
|
|||
"github.com/rcourtman/pulse-go-rewrite/internal/auth"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/tlsutil"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
|
|
@ -101,6 +102,7 @@ type Config struct {
|
|||
GuestMetadataRefreshJitter time.Duration `envconfig:"GUEST_METADATA_REFRESH_JITTER" default:"45s" json:"guestMetadataRefreshJitter"`
|
||||
GuestMetadataRetryBackoff time.Duration `envconfig:"GUEST_METADATA_RETRY_BACKOFF" default:"30s" json:"guestMetadataRetryBackoff"`
|
||||
GuestMetadataMaxConcurrent int `envconfig:"GUEST_METADATA_MAX_CONCURRENT" default:"4" json:"guestMetadataMaxConcurrent"`
|
||||
DNSCacheTimeout time.Duration `envconfig:"DNS_CACHE_TIMEOUT" default:"5m" json:"dnsCacheTimeout"`
|
||||
|
||||
// Logging settings
|
||||
LogLevel string `envconfig:"LOG_LEVEL" default:"info"`
|
||||
|
|
@ -521,6 +523,7 @@ func Load() (*Config, error) {
|
|||
GuestMetadataRefreshJitter: DefaultGuestMetadataRefreshJitter,
|
||||
GuestMetadataRetryBackoff: DefaultGuestMetadataRetryBackoff,
|
||||
GuestMetadataMaxConcurrent: DefaultGuestMetadataMaxConcurrent,
|
||||
DNSCacheTimeout: 5 * time.Minute, // Default DNS cache timeout
|
||||
LogLevel: "info",
|
||||
LogFormat: "auto",
|
||||
LogMaxSize: 100,
|
||||
|
|
@ -615,10 +618,15 @@ func Load() (*Config, error) {
|
|||
}
|
||||
cfg.Discovery = NormalizeDiscoveryConfig(CloneDiscoveryConfig(systemSettings.DiscoveryConfig))
|
||||
cfg.TemperatureMonitoringEnabled = systemSettings.TemperatureMonitoringEnabled
|
||||
// Load DNS cache timeout
|
||||
if systemSettings.DNSCacheTimeout > 0 {
|
||||
cfg.DNSCacheTimeout = time.Duration(systemSettings.DNSCacheTimeout) * time.Second
|
||||
}
|
||||
// APIToken no longer loaded from system.json - only from .env
|
||||
log.Info().
|
||||
Str("updateChannel", cfg.UpdateChannel).
|
||||
Str("logLevel", cfg.LogLevel).
|
||||
Dur("dnsCacheTimeout", cfg.DNSCacheTimeout).
|
||||
Msg("Loaded system configuration")
|
||||
} else {
|
||||
// No system.json exists - create default one
|
||||
|
|
@ -816,6 +824,20 @@ func Load() (*Config, error) {
|
|||
}
|
||||
}
|
||||
|
||||
if dnsCacheTimeout := utils.GetenvTrim("DNS_CACHE_TIMEOUT"); dnsCacheTimeout != "" {
|
||||
if dur, err := time.ParseDuration(dnsCacheTimeout); err == nil {
|
||||
if dur <= 0 {
|
||||
log.Warn().Str("value", dnsCacheTimeout).Msg("Ignoring non-positive DNS_CACHE_TIMEOUT from environment")
|
||||
} else {
|
||||
cfg.DNSCacheTimeout = dur
|
||||
cfg.EnvOverrides["DNS_CACHE_TIMEOUT"] = true
|
||||
log.Info().Dur("timeout", dur).Msg("DNS cache timeout overridden by environment")
|
||||
}
|
||||
} else {
|
||||
log.Warn().Str("value", dnsCacheTimeout).Msg("Invalid DNS_CACHE_TIMEOUT value, expected duration string")
|
||||
}
|
||||
}
|
||||
|
||||
// Support both FRONTEND_PORT (preferred) and PORT (legacy) env vars
|
||||
if frontendPort := os.Getenv("FRONTEND_PORT"); frontendPort != "" {
|
||||
if p, err := strconv.Atoi(frontendPort); err == nil {
|
||||
|
|
@ -1193,6 +1215,10 @@ func Load() (*Config, error) {
|
|||
Component: "pulse-config",
|
||||
})
|
||||
|
||||
// Initialize DNS cache with configured timeout
|
||||
// This must be done before any HTTP clients are created
|
||||
tlsutil.SetDNSCacheTTL(cfg.DNSCacheTimeout)
|
||||
|
||||
// Validate configuration
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("load config: invalid configuration: %w", err)
|
||||
|
|
@ -1230,6 +1256,7 @@ func SaveConfig(cfg *Config) error {
|
|||
AdaptivePollingBaseInterval: int(cfg.AdaptivePollingBaseInterval / time.Second),
|
||||
AdaptivePollingMinInterval: int(cfg.AdaptivePollingMinInterval / time.Second),
|
||||
AdaptivePollingMaxInterval: int(cfg.AdaptivePollingMaxInterval / time.Second),
|
||||
DNSCacheTimeout: int(cfg.DNSCacheTimeout / time.Second),
|
||||
// APIToken removed - now handled via .env only
|
||||
}
|
||||
if err := globalPersistence.SaveSystemSettings(systemSettings); err != nil {
|
||||
|
|
|
|||
|
|
@ -818,9 +818,10 @@ type SystemSettings struct {
|
|||
DiscoverySubnet string `json:"discoverySubnet,omitempty"`
|
||||
DiscoveryConfig DiscoveryConfig `json:"discoveryConfig"`
|
||||
Theme string `json:"theme,omitempty"` // User theme preference: "light", "dark", or empty for system default
|
||||
AllowEmbedding bool `json:"allowEmbedding"` // Allow iframe embedding
|
||||
AllowedEmbedOrigins string `json:"allowedEmbedOrigins,omitempty"` // Comma-separated list of allowed origins for embedding
|
||||
TemperatureMonitoringEnabled bool `json:"temperatureMonitoringEnabled"`
|
||||
AllowEmbedding bool `json:"allowEmbedding"` // Allow iframe embedding
|
||||
AllowedEmbedOrigins string `json:"allowedEmbedOrigins,omitempty"` // Comma-separated list of allowed origins for embedding
|
||||
TemperatureMonitoringEnabled bool `json:"temperatureMonitoringEnabled"`
|
||||
DNSCacheTimeout int `json:"dnsCacheTimeout,omitempty"` // DNS cache timeout in seconds (0 = default 5 minutes)
|
||||
// APIToken removed - now handled via .env file only
|
||||
}
|
||||
|
||||
|
|
@ -828,14 +829,15 @@ type SystemSettings struct {
|
|||
func DefaultSystemSettings() *SystemSettings {
|
||||
defaultDiscovery := DefaultDiscoveryConfig()
|
||||
return &SystemSettings{
|
||||
PBSPollingInterval: 60,
|
||||
PMGPollingInterval: 60,
|
||||
AutoUpdateEnabled: false,
|
||||
DiscoveryEnabled: false,
|
||||
DiscoverySubnet: "auto",
|
||||
DiscoveryConfig: defaultDiscovery,
|
||||
AllowEmbedding: false,
|
||||
PBSPollingInterval: 60,
|
||||
PMGPollingInterval: 60,
|
||||
AutoUpdateEnabled: false,
|
||||
DiscoveryEnabled: false,
|
||||
DiscoverySubnet: "auto",
|
||||
DiscoveryConfig: defaultDiscovery,
|
||||
AllowEmbedding: false,
|
||||
TemperatureMonitoringEnabled: true,
|
||||
DNSCacheTimeout: 300, // Default: 5 minutes (300 seconds)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
100
pkg/tlsutil/dnscache.go
Normal file
100
pkg/tlsutil/dnscache.go
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
package tlsutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/dnscache"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
var (
|
||||
// Global DNS resolver with caching
|
||||
globalResolver *dnscache.Resolver
|
||||
globalResolverOnce sync.Once
|
||||
resolverMutex sync.RWMutex
|
||||
resolverRefreshTTL time.Duration = 5 * time.Minute // Default TTL
|
||||
)
|
||||
|
||||
// GetDNSResolver returns the global DNS resolver instance with caching
|
||||
func GetDNSResolver() *dnscache.Resolver {
|
||||
globalResolverOnce.Do(func() {
|
||||
initDNSResolver(resolverRefreshTTL)
|
||||
})
|
||||
return globalResolver
|
||||
}
|
||||
|
||||
// initDNSResolver initializes the DNS resolver with the specified TTL
|
||||
func initDNSResolver(ttl time.Duration) {
|
||||
log.Info().
|
||||
Dur("ttl", ttl).
|
||||
Msg("Initializing DNS resolver cache to reduce DNS query load")
|
||||
|
||||
globalResolver = &dnscache.Resolver{}
|
||||
|
||||
// Start a goroutine to periodically refresh the DNS cache
|
||||
// This prevents stale DNS entries while still providing caching benefits
|
||||
go func() {
|
||||
ticker := time.NewTicker(ttl)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
globalResolver.Refresh(true)
|
||||
log.Debug().
|
||||
Dur("ttl", ttl).
|
||||
Msg("DNS cache refreshed")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// SetDNSCacheTTL updates the DNS cache TTL
|
||||
// This function should be called before any HTTP clients are created
|
||||
func SetDNSCacheTTL(ttl time.Duration) {
|
||||
resolverMutex.Lock()
|
||||
defer resolverMutex.Unlock()
|
||||
|
||||
if ttl <= 0 {
|
||||
ttl = 5 * time.Minute // Default
|
||||
}
|
||||
|
||||
resolverRefreshTTL = ttl
|
||||
|
||||
log.Info().
|
||||
Dur("ttl", ttl).
|
||||
Msg("DNS cache TTL configured")
|
||||
}
|
||||
|
||||
// DialContextWithCache is a DialContext function that uses the DNS cache
|
||||
func DialContextWithCache(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
resolver := GetDNSResolver()
|
||||
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Look up the IP address using the cached resolver
|
||||
ips, err := resolver.LookupHost(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Use the first IP address
|
||||
if len(ips) == 0 {
|
||||
return nil, &net.DNSError{
|
||||
Err: "no IP addresses found",
|
||||
Name: host,
|
||||
}
|
||||
}
|
||||
|
||||
// Create a dialer with the resolved IP
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
|
||||
// Dial with the resolved IP address
|
||||
return dialer.DialContext(ctx, network, net.JoinHostPort(ips[0], port))
|
||||
}
|
||||
|
|
@ -6,7 +6,6 @@ import (
|
|||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -53,12 +52,9 @@ func CreateHTTPClientWithTimeout(verifySSL bool, fingerprint string, timeout tim
|
|||
MaxConnsPerHost: 20, // Limit concurrent connections per host
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: true, // Disable compression for lower latency
|
||||
// Add specific timeouts for DNS, TLS handshake, and response headers
|
||||
// These prevent hanging on DNS resolution or TLS negotiation
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 10 * time.Second, // Connection timeout
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
// Use DNS caching to reduce DNS queries
|
||||
// This prevents excessive DNS lookups for frequently accessed Proxmox hosts
|
||||
DialContext: DialContextWithCache,
|
||||
TLSHandshakeTimeout: 10 * time.Second, // TLS handshake timeout
|
||||
ResponseHeaderTimeout: 10 * time.Second, // Time to wait for response headers
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue