Merge pull request #1183 from safing/feature/use-stale-dns-cache-and-update-server-flag

Use stale DNS cache entries and update server flag

cmds/portmaster-start/main.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -24,11 +25,13 @@ import (
 
 var (
 	dataDir    string
-	staging    bool
 	maxRetries int
 	dataRoot   *utils.DirStructure
 	logsRoot   *utils.DirStructure
 
+	updateURLFlag string
+	userAgentFlag string
+
 	// Create registry.
 	registry = &updater.ResourceRegistry{
 		Name: "updates",
@@ -67,8 +70,8 @@ func init() {
 	flags := rootCmd.PersistentFlags()
 	{
 		flags.StringVar(&dataDir, "data", "", "Configures the data directory. Alternatively, this can also be set via the environment variable PORTMASTER_DATA.")
-		flags.StringVar(&registry.UserAgent, "update-agent", "Start", "Sets the user agent for requests to the update server")
+		flags.StringVar(&updateURLFlag, "update-server", "", "Set an alternative update server (full URL)")
-		flags.BoolVar(&staging, "staging", false, "Deprecated, configure in settings instead.")
+		flags.StringVar(&userAgentFlag, "update-agent", "", "Set an alternative user agent for requests to the update server")
 		flags.IntVar(&maxRetries, "max-retries", 5, "Maximum number of retries when starting a Portmaster component")
 		flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdin.")
 		_ = rootCmd.MarkPersistentFlagDirname("data")
@@ -137,6 +140,25 @@ func initCobra() {
 }
 
 func configureRegistry(mustLoadIndex bool) error {
+	// Check if update server URL supplied via flag is a valid URL.
+	if updateURLFlag != "" {
+		u, err := url.Parse(updateURLFlag)
+		if err != nil {
+			return fmt.Errorf("supplied update server URL is invalid: %w", err)
+		}
+		if u.Scheme != "https" {
+			return errors.New("supplied update server URL must use HTTPS")
+		}
+	}
+
+	// Override values from flags.
+	if userAgentFlag != "" {
+		registry.UserAgent = userAgentFlag
+	}
+	if updateURLFlag != "" {
+		registry.UpdateURLs = []string{updateURLFlag}
+	}
+
 	// If dataDir is not set, check the environment variable.
 	if dataDir == "" {
 		dataDir = os.Getenv("PORTMASTER_DATA")

cmds/portmaster-start/update.go

@@ -134,10 +134,8 @@ func logProgress(state *updater.RegistryState) {
 					len(downloadDetails.Resources),
 					downloadDetails.Resources[downloadDetails.FinishedUpTo],
 				)
-			} else {
+			} else if state.Updates.LastDownloadAt == nil {
-				if state.Updates.LastDownloadAt == nil {
+				log.Println("finalizing downloads")
-					log.Println("finalizing downloads")
-				}
 			}
 		}
 	}

profile/config.go

@@ -270,6 +270,8 @@ Examples:
 - "1.1.1.1 ICMP"
 
 Important: DNS Requests are only matched against domain and filter list rules, all others require an IP address and are checked only with the following IP connection.
+
+Pro Tip: You can use "#" to add a comment to a rule.
 `, `"`, "`")
 
 	// rulesVerdictNames defines the verdicts names to be used for filter rules.

resolver/config.go

@@ -62,13 +62,17 @@ var (
 	noAssignedNameservers               status.SecurityLevelOptionFunc
 	cfgOptionNoAssignedNameserversOrder = 1
 
+	CfgOptionUseStaleCacheKey   = "dns/useStaleCache"
+	useStaleCache               config.BoolOption
+	cfgOptionUseStaleCacheOrder = 2
+
 	CfgOptionNoMulticastDNSKey   = "dns/noMulticastDNS"
 	noMulticastDNS               status.SecurityLevelOptionFunc
-	cfgOptionNoMulticastDNSOrder = 2
+	cfgOptionNoMulticastDNSOrder = 3
 
 	CfgOptionNoInsecureProtocolsKey   = "dns/noInsecureProtocols"
 	noInsecureProtocols               status.SecurityLevelOptionFunc
-	cfgOptionNoInsecureProtocolsOrder = 3
+	cfgOptionNoInsecureProtocolsOrder = 4
 
 	CfgOptionDontResolveSpecialDomainsKey   = "dns/dontResolveSpecialDomains"
 	dontResolveSpecialDomains               status.SecurityLevelOptionFunc
@@ -161,11 +165,11 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 	configuredNameServers = config.Concurrent.GetAsStringArray(CfgOptionNameServersKey, defaultNameServers)
 
 	err = config.Register(&config.Option{
-		Name:           "Retry Timeout",
+		Name:           "Ignore Failing DNS Servers Duration",
 		Key:            CfgOptionNameserverRetryRateKey,
-		Description:    "Timeout between retries when a DNS server fails.",
+		Description:    "Duration in seconds how long a failing DNS server should not be retried.",
 		OptType:        config.OptTypeInt,
-		ExpertiseLevel: config.ExpertiseLevelExpert,
+		ExpertiseLevel: config.ExpertiseLevelDeveloper,
 		ReleaseLevel:   config.ReleaseLevelStable,
 		DefaultValue:   300,
 		Annotations: config.Annotations{
@@ -201,6 +205,24 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 	}
 	noAssignedNameservers = status.SecurityLevelOption(CfgOptionNoAssignedNameserversKey)
 
+	err = config.Register(&config.Option{
+		Name:           "Always Use DNS Cache",
+		Key:            CfgOptionUseStaleCacheKey,
+		Description:    "Always use stale entries from the DNS cache and refresh expired entries afterwards. This can improve DNS resolving performance a lot, but may lead to occasional connection errors due to the outdated DNS records.",
+		OptType:        config.OptTypeBool,
+		ExpertiseLevel: config.ExpertiseLevelUser,
+		ReleaseLevel:   config.ReleaseLevelStable,
+		DefaultValue:   false,
+		Annotations: config.Annotations{
+			config.DisplayOrderAnnotation: cfgOptionUseStaleCacheOrder,
+			config.CategoryAnnotation:     "Resolving",
+		},
+	})
+	if err != nil {
+		return err
+	}
+	useStaleCache = config.Concurrent.GetAsBool(CfgOptionUseStaleCacheKey, false)
+
 	err = config.Register(&config.Option{
 		Name:           "Ignore Multicast DNS",
 		Key:            CfgOptionNoMulticastDNSKey,

resolver/resolve.go

@@ -179,8 +179,21 @@ func Resolve(ctx context.Context, q *Query) (rrCache *RRCache, err error) {
 	// check the cache
 	if !q.NoCaching {
 		rrCache = checkCache(ctx, q)
-		if rrCache != nil && !rrCache.Expired() {
+		if rrCache != nil {
-			return rrCache, nil
+			switch {
+			case !rrCache.Expired():
+				// Return non-expired cached entry immediately.
+				return rrCache, nil
+			case useStaleCache():
+				// Return expired cache if we should use stale cache entries,
+				// but start an async query instead.
+				log.Tracer(ctx).Tracef(
+					"resolver: using stale cache entry that expired %s ago",
+					time.Since(time.Unix(rrCache.Expires, 0)).Round(time.Second),
+				)
+				startAsyncQuery(ctx, q, rrCache)
+				return rrCache, nil
+			}
 		}
 
 		// dedupe!
@@ -188,7 +201,9 @@ func Resolve(ctx context.Context, q *Query) (rrCache *RRCache, err error) {
 		if markRequestFinished == nil {
 			// we waited for another request, recheck the cache!
 			rrCache = checkCache(ctx, q)
-			if rrCache != nil && !rrCache.Expired() {
+			if rrCache != nil && (!rrCache.Expired() || useStaleCache()) {
+				// Return non-expired or expired entry if we should use stale cache entries.
+				// There just was a request, so do not trigger an async query.
 				return rrCache, nil
 			}
 			log.Tracer(ctx).Debugf("resolver: waited for another %s%s query, but cache missed!", q.FQDN, q.QType)
@@ -232,63 +247,71 @@ func checkCache(ctx context.Context, q *Query) *RRCache {
 		return nil
 	}
 
-	// Check if we want to reset the cache for this entry.
+	switch {
-	if shouldResetCache(q) {
+	case shouldResetCache(q):
+		// Check if we want to reset the cache for this entry.
 		err := ResetCachedRecord(q.FQDN, q.QType.String())
 		switch {
 		case err == nil:
-			log.Tracer(ctx).Tracef("resolver: cache for %s%s was reset", q.FQDN, q.QType)
+			log.Tracer(ctx).Infof("resolver: cache for %s%s was reset", q.FQDN, q.QType)
 		case errors.Is(err, database.ErrNotFound):
 			log.Tracer(ctx).Tracef("resolver: cache for %s%s was already reset (is empty)", q.FQDN, q.QType)
 		default:
 			log.Tracer(ctx).Warningf("resolver: failed to reset cache for %s%s: %s", q.FQDN, q.QType, err)
 		}
 		return nil
-	}
 
-	// Check if the cache has already expired.
+	case rrCache.Expired():
-	// We still return the cache, if it isn't NXDomain, as it will be used if the
+		// Check if the cache has already expired.
-	// new query fails.
+		// We still return the cache, if it isn't NXDomain, as it will be used if the
-	if rrCache.Expired() {
+		// new query fails.
 		if rrCache.RCode == dns.RcodeSuccess {
 			return rrCache
 		}
 		return nil
-	}
 
-	// Check if the cache will expire soon and start an async request.
+	case rrCache.ExpiresSoon():
-	if rrCache.ExpiresSoon() {
+		// Check if the cache will expire soon and start an async request.
-		// Set flag that we are refreshing this entry.
+		startAsyncQuery(ctx, q, rrCache)
-		rrCache.RequestingNew = true
+		return rrCache
 
+	default:
+		// Return still valid cache entry.
 		log.Tracer(ctx).Tracef(
-			"resolver: cache for %s will expire in %s, refreshing async now",
+			"resolver: using cached RR (expires in %s)",
-			q.ID(),
 			time.Until(time.Unix(rrCache.Expires, 0)).Round(time.Second),
 		)
-
-		// resolve async
-		module.StartWorker("resolve async", func(asyncCtx context.Context) error {
-			tracingCtx, tracer := log.AddTracer(asyncCtx)
-			defer tracer.Submit()
-			tracer.Tracef("resolver: resolving %s async", q.ID())
-			_, err := resolveAndCache(tracingCtx, q, nil)
-			if err != nil {
-				tracer.Warningf("resolver: async query for %s failed: %s", q.ID(), err)
-			} else {
-				tracer.Infof("resolver: async query for %s succeeded", q.ID())
-			}
-			return nil
-		})
-
 		return rrCache
 	}
+}
 
+func startAsyncQuery(ctx context.Context, q *Query, currentRRCache *RRCache) {
+	// Check if an async query was already started.
+	if currentRRCache.RequestingNew {
+		return
+	}
+
+	// Set flag and log that we are refreshing this entry.
+	currentRRCache.RequestingNew = true
 	log.Tracer(ctx).Tracef(
-		"resolver: using cached RR (expires in %s)",
+		"resolver: cache for %s will expire in %s, refreshing async now",
-		time.Until(time.Unix(rrCache.Expires, 0)).Round(time.Second),
+		q.ID(),
+		time.Until(time.Unix(currentRRCache.Expires, 0)).Round(time.Second),
 	)
-	return rrCache
+
+	// resolve async
+	module.StartWorker("resolve async", func(asyncCtx context.Context) error {
+		tracingCtx, tracer := log.AddTracer(asyncCtx)
+		defer tracer.Submit()
+		tracer.Tracef("resolver: resolving %s async", q.ID())
+		_, err := resolveAndCache(tracingCtx, q, nil)
+		if err != nil {
+			tracer.Warningf("resolver: async query for %s failed: %s", q.ID(), err)
+		} else {
+			tracer.Infof("resolver: async query for %s succeeded", q.ID())
+		}
+		return nil
+	})
 }
 
 func deduplicateRequest(ctx context.Context, q *Query) (finishRequest func()) {

resolver/rrcache.go

@@ -55,7 +55,8 @@ func (rrCache *RRCache) Expired() bool {
 	return rrCache.Expires <= time.Now().Unix()
 }
 
-// ExpiresSoon returns whether the record will expire soon and should already be refreshed.
+// ExpiresSoon returns whether the record will expire soon (or already has) and
+// should already be refreshed.
 func (rrCache *RRCache) ExpiresSoon() bool {
 	return rrCache.Expires <= time.Now().Unix()+refreshTTL
 }

updates/main.go

@@ -2,8 +2,10 @@ package updates
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"net/url"
 	"runtime"
 	"time"
 
@@ -41,9 +43,11 @@ const (
 )
 
 var (
-	module            *modules.Module
+	module   *modules.Module
-	registry          *updater.ResourceRegistry
+	registry *updater.ResourceRegistry
-	userAgentFromFlag string
+
+	userAgentFromFlag    string
+	updateServerFromFlag string
 
 	updateTask          *modules.Task
 	updateASAP          bool
@@ -59,6 +63,11 @@ var (
 	// fetching resources from the update server.
 	UserAgent = fmt.Sprintf("Portmaster (%s %s)", runtime.GOOS, runtime.GOARCH)
 
+	// DefaultUpdateURLs defines the default base URLs of the update server.
+	DefaultUpdateURLs = []string{
+		"https://updates.safing.io",
+	}
+
 	// DisableSoftwareAutoUpdate specifies whether software updates should be disabled.
 	// This is used on Android, as it will never require binary updates.
 	DisableSoftwareAutoUpdate = false
@@ -75,10 +84,8 @@ func init() {
 	module.RegisterEvent(VersionUpdateEvent, true)
 	module.RegisterEvent(ResourceUpdateEvent, true)
 
-	flag.StringVar(&userAgentFromFlag, "update-agent", "", "set the user agent for requests to the update server")
+	flag.StringVar(&updateServerFromFlag, "update-server", "", "set an alternative update server (full URL)")
-
+	flag.StringVar(&userAgentFromFlag, "update-agent", "", "set an alternative user agent for requests to the update server")
-	var dummy bool
-	flag.BoolVar(&dummy, "staging", false, "deprecated, configure in settings instead")
 }
 
 func prep() error {
@@ -86,6 +93,17 @@ func prep() error {
 		return err
 	}
 
+	// Check if update server URL supplied via flag is a valid URL.
+	if updateServerFromFlag != "" {
+		u, err := url.Parse(updateServerFromFlag)
+		if err != nil {
+			return fmt.Errorf("supplied update server URL is invalid: %w", err)
+		}
+		if u.Scheme != "https" {
+			return errors.New("supplied update server URL must use HTTPS")
+		}
+	}
+
 	return registerAPIEndpoints()
 }
 
@@ -104,10 +122,8 @@ func start() error {
 
 	// create registry
 	registry = &updater.ResourceRegistry{
-		Name: ModuleName,
+		Name:             ModuleName,
-		UpdateURLs: []string{
+		UpdateURLs:       DefaultUpdateURLs,
-			"https://updates.safing.io",
-		},
 		UserAgent:        UserAgent,
 		MandatoryUpdates: helper.MandatoryUpdates(),
 		AutoUnpack:       helper.AutoUnpackUpdates(),
@@ -115,10 +131,13 @@ func start() error {
 		DevMode:          devMode(),
 		Online:           true,
 	}
+	// Override values from flags.
 	if userAgentFromFlag != "" {
-		// override with flag value
 		registry.UserAgent = userAgentFromFlag
 	}
+	if updateServerFromFlag != "" {
+		registry.UpdateURLs = []string{updateServerFromFlag}
+	}
 
 	// pre-init state
 	updateStateExport, err := LoadStateExport()