Improve history purging

netquery/database.go

@@ -16,6 +16,7 @@ import (
 	"zombiezen.com/go/sqlite"
 	"zombiezen.com/go/sqlite/sqlitex"
 
+	"github.com/safing/portbase/config"
 	"github.com/safing/portbase/dataroot"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portmaster/netquery/orm"
@@ -203,6 +204,7 @@ func NewInMemory() (*Database, error) {
 	return db, nil
 }
 
+// Close closes the database, including pools and connections.
 func (db *Database) Close() error {
 	db.readConnPool.Close()
 
@@ -213,7 +215,8 @@ func (db *Database) Close() error {
 	return nil
 }
 
-func VacuumHistory(ctx context.Context) error {
+// VacuumHistory rewrites the history database in order to purge deleted records.
+func VacuumHistory(ctx context.Context) (err error) {
 	historyParentDir := dataroot.Root().ChildDir("databases", 0o700)
 	if err := historyParentDir.Ensure(); err != nil {
 		return fmt.Errorf("failed to ensure database directory exists: %w", err)
@@ -235,6 +238,11 @@ func VacuumHistory(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if closeErr := writeConn.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+	}()
 
 	return orm.RunQuery(ctx, writeConn, "VACUUM")
 }
@@ -414,50 +422,72 @@ func (db *Database) dumpTo(ctx context.Context, w io.Writer) error { //nolint:un
 	return enc.Encode(conns)
 }
 
-func (db *Database) CleanupHistoryData(ctx context.Context) error {
+// PurgeOldHistory deletes history data outside of the (per-app) retention time frame.
-	query := "SELECT DISTINCT profile FROM history.connections"
+func (db *Database) PurgeOldHistory(ctx context.Context) error {
+	// Setup tracer for the clean up process.
+	ctx, tracer := log.AddTracer(ctx)
+	defer tracer.Submit()
+	defer tracer.Info("history: deleted connections outside of retention from %d profiles")
 
+	// Get list of profiles in history.
+	query := "SELECT DISTINCT profile FROM history.connections"
 	var result []struct {
 		Profile string `sqlite:"profile"`
 	}
-
 	if err := db.Execute(ctx, query, orm.WithResult(&result)); err != nil {
 		return fmt.Errorf("failed to get a list of profiles from the history database: %w", err)
 	}
 
-	globalRetentionDays := profile.CfgOptionHistoryRetention()
+	var (
-	merr := new(multierror.Error)
+		// Get global retention days - do not delete in case of error.
+		globalRetentionDays = config.GetAsInt(profile.CfgOptionKeepHistoryKey, 0)()
 
+		profileName   string
+		retentionDays int64
+
+		profileCnt int
+		merr       = new(multierror.Error)
+	)
 	for _, row := range result {
+		// Get profile and retention days.
 		id := strings.TrimPrefix(row.Profile, string(profile.SourceLocal)+"/")
 		p, err := profile.GetLocalProfile(id, nil, nil)
-
-		var retention int
 		if err == nil {
-			retention = p.HistoryRetention()
+			profileName = p.String()
+			retentionDays = p.LayeredProfile().KeepHistory()
 		} else {
-			// we failed to get the profile, fallback to the global setting
+			// Getting profile failed, fallback to global setting.
-			log.Errorf("failed to load profile for id %s: %s", id, err)
+			tracer.Errorf("history: failed to load profile for id %s: %s", id, err)
-			retention = int(globalRetentionDays)
+			profileName = row.Profile
+			retentionDays = globalRetentionDays
 		}
 
-		if retention == 0 {
+		// Skip deleting if history should be kept forever.
-			log.Infof("skipping history data retention for profile %s: retention is disabled", row.Profile)
+		if retentionDays == 0 {
-
+			tracer.Tracef("history: retention is disabled for %s, skipping", profileName)
 			continue
 		}
+		// Count profiles where connections were deleted.
+		profileCnt++
 
-		threshold := time.Now().Add(-1 * time.Duration(retention) * time.Hour * 24)
+		// TODO: count cleared connections
-
+		threshold := time.Now().Add(-1 * time.Duration(retentionDays) * time.Hour * 24)
-		log.Infof("cleaning up history data for profile %s with retention setting %d days (threshold = %s)", row.Profile, retention, threshold.Format(orm.SqliteTimeFormat))
+		if err := db.ExecuteWrite(ctx,
-
+			"DELETE FROM history.connections WHERE profile = :profile AND active = FALSE AND datetime(started) < datetime(:threshold)",
-		query := "DELETE FROM history.connections WHERE profile = :profile AND active = FALSE AND datetime(started) < datetime(:threshold)"
+			orm.WithNamedArgs(map[string]any{
-		if err := db.ExecuteWrite(ctx, query, orm.WithNamedArgs(map[string]any{
+				":profile":   row.Profile,
-			":profile":   row.Profile,
+				":threshold": threshold.Format(orm.SqliteTimeFormat),
-			":threshold": threshold.Format(orm.SqliteTimeFormat),
+			}),
-		})); err != nil {
+		); err != nil {
-			log.Errorf("failed to delete connections for profile %s from history: %s", row.Profile, err)
+			tracer.Warningf("history: failed to delete connections of %s: %s", profileName, err)
 			merr.Errors = append(merr.Errors, fmt.Errorf("profile %s: %w", row.Profile, err))
+		} else {
+			tracer.Debugf(
+				"history: deleted connections older than %d days (before %s) of %s",
+				retentionDays,
+				threshold,
+				profileName,
+			)
 		}
 	}
 

netquery/manager.go

@@ -40,8 +40,8 @@ type (
 		// the bandwidth data to the history database.
 		UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, bytesReceived uint64, bytesSent uint64) error
 
-		// CleanupHistoryData applies data retention to the history database.
+		// PurgeOldHistory deletes data outside of the retention time frame from the history database.
-		CleanupHistoryData(ctx context.Context) error
+		PurgeOldHistory(ctx context.Context) error
 
 		// Close closes the connection store. It must not be used afterwards.
 		Close() error

netquery/module_api.go

@@ -87,35 +87,37 @@ func (m *module) prepare() error {
 	}
 
 	if err := api.RegisterEndpoint(api.Endpoint{
+		Name:        "Query Connections",
+		Description: "Query the in-memory sqlite connection database.",
 		Path:        "netquery/query",
 		MimeType:    "application/json",
 		Read:        api.PermitUser, // Needs read+write as the query is sent using POST data.
 		Write:       api.PermitUser, // Needs read+write as the query is sent using POST data.
 		BelongsTo:   m.Module,
 		HandlerFunc: queryHander.ServeHTTP,
-		Name:        "Query Connections",
-		Description: "Query the in-memory sqlite connection database.",
 	}); err != nil {
 		return fmt.Errorf("failed to register API endpoint: %w", err)
 	}
 
 	if err := api.RegisterEndpoint(api.Endpoint{
+		Name:        "Active Connections Chart",
+		Description: "Query the in-memory sqlite connection database and return a chart of active connections.",
 		Path:        "netquery/charts/connection-active",
 		MimeType:    "application/json",
 		Write:       api.PermitUser,
 		BelongsTo:   m.Module,
 		HandlerFunc: chartHandler.ServeHTTP,
-		Name:        "Active Connections Chart",
-		Description: "Query the in-memory sqlite connection database and return a chart of active connections.",
 	}); err != nil {
 		return fmt.Errorf("failed to register API endpoint: %w", err)
 	}
 
 	if err := api.RegisterEndpoint(api.Endpoint{
-		Path:      "netquery/history/clear",
+		Name:        "Remove connections from profile history",
-		MimeType:  "application/json",
+		Description: "Remove all connections from the history database for one or more profiles",
-		Write:     api.PermitUser,
+		Path:        "netquery/history/clear",
-		BelongsTo: m.Module,
+		MimeType:    "application/json",
+		Write:       api.PermitUser,
+		BelongsTo:   m.Module,
 		HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
 			var body struct {
 				ProfileIDs []string `json:"profileIDs"`
@@ -154,28 +156,21 @@ func (m *module) prepare() error {
 
 			w.WriteHeader(http.StatusNoContent)
 		},
-		Name:        "Remove connections from profile history",
-		Description: "Remove all connections from the history database for one or more profiles",
 	}); err != nil {
 		return fmt.Errorf("failed to register API endpoint: %w", err)
-
 	}
 
 	if err := api.RegisterEndpoint(api.Endpoint{
+		Name:      "Apply connection history retention threshold",
 		Path:      "netquery/history/cleanup",
-		MimeType:  "application/json",
 		Write:     api.PermitUser,
 		BelongsTo: m.Module,
-		HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
+		ActionFunc: func(ar *api.Request) (msg string, err error) {
-			if err := m.Store.CleanupHistoryData(r.Context()); err != nil {
+			if err := m.Store.PurgeOldHistory(ar.Context()); err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return "", err
-
-				return
 			}
-
+			return "Deleted outdated connections.", nil
-			w.WriteHeader(http.StatusNoContent)
 		},
-		Name: "Apply connection history retention threshold",
 	}); err != nil {
 		return fmt.Errorf("failed to register API endpoint: %w", err)
 	}
@@ -184,7 +179,7 @@ func (m *module) prepare() error {
 }
 
 func (m *module) start() error {
-	m.StartServiceWorker("netquery-feeder", time.Second, func(ctx context.Context) error {
+	m.StartServiceWorker("netquery connection feed listener", 0, func(ctx context.Context) error {
 		sub, err := m.db.Subscribe(query.New("network:"))
 		if err != nil {
 			return fmt.Errorf("failed to subscribe to network tree: %w", err)
@@ -215,16 +210,12 @@ func (m *module) start() error {
 		}
 	})
 
-	m.StartServiceWorker("netquery-persister", time.Second, func(ctx context.Context) error {
+	m.StartServiceWorker("netquery connection feed handler", 0, func(ctx context.Context) error {
 		m.mng.HandleFeed(ctx, m.feed)
 		return nil
 	})
 
-	m.StartServiceWorker("history-row-cleaner", time.Hour, func(ctx context.Context) error {
+	m.StartServiceWorker("netquery live db cleaner", 0, func(ctx context.Context) error {
-		return m.Store.CleanupHistoryData(ctx)
-	})
-
-	m.StartServiceWorker("netquery-row-cleaner", time.Second, func(ctx context.Context) error {
 		for {
 			select {
 			case <-ctx.Done():
@@ -233,14 +224,18 @@ func (m *module) start() error {
 				threshold := time.Now().Add(-network.DeleteConnsAfterEndedThreshold)
 				count, err := m.Store.Cleanup(ctx, threshold)
 				if err != nil {
-					log.Errorf("netquery: failed to count number of rows in memory: %s", err)
+					log.Errorf("netquery: failed to removed old connections from live db: %s", err)
 				} else {
-					log.Tracef("netquery: successfully removed %d old rows that ended before %s", count, threshold)
+					log.Tracef("netquery: successfully removed %d old connections from live db that ended before %s", count, threshold)
 				}
 			}
 		}
 	})
 
+	m.NewTask("network history cleaner", func(ctx context.Context, _ *modules.Task) error {
+		return m.Store.PurgeOldHistory(ctx)
+	}).Repeat(time.Hour).Schedule(time.Now().Add(10 * time.Minute))
+
 	// For debugging, provide a simple direct SQL query interface using
 	// the runtime database.
 	// Only expose in development mode.
@@ -269,9 +264,9 @@ func (m *module) stop() error {
 	if err := m.mng.store.Close(); err != nil {
 		log.Errorf("netquery: failed to close sqlite database: %s", err)
 	} else {
-		// try to vaccum the history database now
+		// Clear deleted connections from database.
 		if err := VacuumHistory(ctx); err != nil {
-			log.Errorf("netquery: failed to execute VACCUM in history database: %s", err)
+			log.Errorf("netquery: failed to execute VACUUM in history database: %s", err)
 		}
 	}
 

network/connection.go

@@ -594,7 +594,7 @@ func (conn *Connection) UpdateFeatures() error {
 	if user.MayUse(account.FeatureHistory) {
 		lProfile := conn.Process().Profile()
 		if lProfile != nil {
-			conn.HistoryEnabled = lProfile.HistoryEnabled()
+			conn.HistoryEnabled = lProfile.EnableHistory()
 		}
 	}
 

profile/config.go

@@ -112,9 +112,9 @@ var (
 	cfgOptionEnableHistory      config.BoolOption
 	cfgOptionEnableHistoryOrder = 96
 
-	CfgOptionHistoryRetentionKey   = "history/retention"
+	CfgOptionKeepHistoryKey   = "history/keep"
-	CfgOptionHistoryRetention      config.IntOption
+	cfgOptionKeepHistory      config.IntOption
-	cfgOptionHistoryRetentionOrder = 97
+	cfgOptionKeepHistoryOrder = 97
 
 	// Setting "Enable SPN" at order 128.
 
@@ -252,7 +252,7 @@ func registerConfiguration() error { //nolint:maintidx
 
 	// Enable History
 	err = config.Register(&config.Option{
-		Name:           "Enable Connection History",
+		Name:           "Enable Network History",
 		Key:            CfgOptionEnableHistoryKey,
 		Description:    "Save connections in a database (on disk) in order to view and search them later. Changes might take a couple minutes to apply to all connections.",
 		OptType:        config.OptTypeBool,
@@ -261,7 +261,7 @@ func registerConfiguration() error { //nolint:maintidx
 		DefaultValue:   false,
 		Annotations: config.Annotations{
 			config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder,
-			config.CategoryAnnotation:     "History",
+			config.CategoryAnnotation:     "General",
 			config.RequiresFeatureID:      account.FeatureHistory,
 		},
 	})
@@ -272,25 +272,29 @@ func registerConfiguration() error { //nolint:maintidx
 	cfgBoolOptions[CfgOptionEnableHistoryKey] = cfgOptionEnableHistory
 
 	err = config.Register(&config.Option{
-		Name:           "History Data Retention",
+		Name: "Keep Network History",
-		Key:            CfgOptionHistoryRetentionKey,
+		Key:  CfgOptionKeepHistoryKey,
-		Description:    "How low, in days, connections should be kept in history.",
+		Description: `Specify how many days the network history data should be kept. Please keep in mind that more available history data makes reports (coming soon) a lot more useful.
+		
+Older data is deleted in intervals and cleared from the database continually. If in a hurry, shutdown or restart Portmaster to clear deleted entries immediately.
+
+Set to 0 days to keep network history forever. Depending on your device, this might affect performance.`,
 		OptType:        config.OptTypeInt,
 		ReleaseLevel:   config.ReleaseLevelStable,
 		ExpertiseLevel: config.ExpertiseLevelUser,
-		DefaultValue:   7,
+		DefaultValue:   30,
 		Annotations: config.Annotations{
 			config.UnitAnnotation:         "Days",
-			config.DisplayOrderAnnotation: cfgOptionHistoryRetentionOrder,
+			config.DisplayOrderAnnotation: cfgOptionKeepHistoryOrder,
-			config.CategoryAnnotation:     "History",
+			config.CategoryAnnotation:     "General",
 			config.RequiresFeatureID:      account.FeatureHistory,
 		},
 	})
 	if err != nil {
 		return err
 	}
-	CfgOptionHistoryRetention = config.Concurrent.GetAsInt(CfgOptionHistoryRetentionKey, 7)
+	cfgOptionKeepHistory = config.Concurrent.GetAsInt(CfgOptionKeepHistoryKey, 30)
-	cfgIntOptions[CfgOptionHistoryRetentionKey] = CfgOptionHistoryRetention
+	cfgIntOptions[CfgOptionKeepHistoryKey] = cfgOptionKeepHistory
 
 	rulesHelp := strings.ReplaceAll(`Rules are checked from top to bottom, stopping after the first match. They can match: