diff --git a/netquery/database.go b/netquery/database.go index d71c4501..a52064cf 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -16,6 +16,7 @@ import ( "zombiezen.com/go/sqlite" "zombiezen.com/go/sqlite/sqlitex" + "github.com/safing/portbase/config" "github.com/safing/portbase/dataroot" "github.com/safing/portbase/log" "github.com/safing/portmaster/netquery/orm" @@ -203,6 +204,7 @@ func NewInMemory() (*Database, error) { return db, nil } +// Close closes the database, including pools and connections. func (db *Database) Close() error { db.readConnPool.Close() @@ -213,7 +215,8 @@ func (db *Database) Close() error { return nil } -func VacuumHistory(ctx context.Context) error { +// VacuumHistory rewrites the history database in order to purge deleted records. +func VacuumHistory(ctx context.Context) (err error) { historyParentDir := dataroot.Root().ChildDir("databases", 0o700) if err := historyParentDir.Ensure(); err != nil { return fmt.Errorf("failed to ensure database directory exists: %w", err) @@ -235,6 +238,11 @@ func VacuumHistory(ctx context.Context) error { if err != nil { return err } + defer func() { + if closeErr := writeConn.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() return orm.RunQuery(ctx, writeConn, "VACUUM") } @@ -414,50 +422,72 @@ func (db *Database) dumpTo(ctx context.Context, w io.Writer) error { //nolint:un return enc.Encode(conns) } -func (db *Database) CleanupHistoryData(ctx context.Context) error { - query := "SELECT DISTINCT profile FROM history.connections" +// PurgeOldHistory deletes history data outside of the (per-app) retention time frame. +func (db *Database) PurgeOldHistory(ctx context.Context) error { + // Setup tracer for the clean up process. + ctx, tracer := log.AddTracer(ctx) + defer tracer.Submit() + defer tracer.Info("history: deleted connections outside of retention from %d profiles") + // Get list of profiles in history. + query := "SELECT DISTINCT profile FROM history.connections" var result []struct { Profile string `sqlite:"profile"` } - if err := db.Execute(ctx, query, orm.WithResult(&result)); err != nil { return fmt.Errorf("failed to get a list of profiles from the history database: %w", err) } - globalRetentionDays := profile.CfgOptionHistoryRetention() - merr := new(multierror.Error) + var ( + // Get global retention days - do not delete in case of error. + globalRetentionDays = config.GetAsInt(profile.CfgOptionKeepHistoryKey, 0)() + profileName string + retentionDays int64 + + profileCnt int + merr = new(multierror.Error) + ) for _, row := range result { + // Get profile and retention days. id := strings.TrimPrefix(row.Profile, string(profile.SourceLocal)+"/") p, err := profile.GetLocalProfile(id, nil, nil) - - var retention int if err == nil { - retention = p.HistoryRetention() + profileName = p.String() + retentionDays = p.LayeredProfile().KeepHistory() } else { - // we failed to get the profile, fallback to the global setting - log.Errorf("failed to load profile for id %s: %s", id, err) - retention = int(globalRetentionDays) + // Getting profile failed, fallback to global setting. + tracer.Errorf("history: failed to load profile for id %s: %s", id, err) + profileName = row.Profile + retentionDays = globalRetentionDays } - if retention == 0 { - log.Infof("skipping history data retention for profile %s: retention is disabled", row.Profile) - + // Skip deleting if history should be kept forever. + if retentionDays == 0 { + tracer.Tracef("history: retention is disabled for %s, skipping", profileName) continue } + // Count profiles where connections were deleted. + profileCnt++ - threshold := time.Now().Add(-1 * time.Duration(retention) * time.Hour * 24) - - log.Infof("cleaning up history data for profile %s with retention setting %d days (threshold = %s)", row.Profile, retention, threshold.Format(orm.SqliteTimeFormat)) - - query := "DELETE FROM history.connections WHERE profile = :profile AND active = FALSE AND datetime(started) < datetime(:threshold)" - if err := db.ExecuteWrite(ctx, query, orm.WithNamedArgs(map[string]any{ - ":profile": row.Profile, - ":threshold": threshold.Format(orm.SqliteTimeFormat), - })); err != nil { - log.Errorf("failed to delete connections for profile %s from history: %s", row.Profile, err) + // TODO: count cleared connections + threshold := time.Now().Add(-1 * time.Duration(retentionDays) * time.Hour * 24) + if err := db.ExecuteWrite(ctx, + "DELETE FROM history.connections WHERE profile = :profile AND active = FALSE AND datetime(started) < datetime(:threshold)", + orm.WithNamedArgs(map[string]any{ + ":profile": row.Profile, + ":threshold": threshold.Format(orm.SqliteTimeFormat), + }), + ); err != nil { + tracer.Warningf("history: failed to delete connections of %s: %s", profileName, err) merr.Errors = append(merr.Errors, fmt.Errorf("profile %s: %w", row.Profile, err)) + } else { + tracer.Debugf( + "history: deleted connections older than %d days (before %s) of %s", + retentionDays, + threshold, + profileName, + ) } } diff --git a/netquery/manager.go b/netquery/manager.go index 857576e9..7a0aae39 100644 --- a/netquery/manager.go +++ b/netquery/manager.go @@ -40,8 +40,8 @@ type ( // the bandwidth data to the history database. UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, bytesReceived uint64, bytesSent uint64) error - // CleanupHistoryData applies data retention to the history database. - CleanupHistoryData(ctx context.Context) error + // PurgeOldHistory deletes data outside of the retention time frame from the history database. + PurgeOldHistory(ctx context.Context) error // Close closes the connection store. It must not be used afterwards. Close() error diff --git a/netquery/module_api.go b/netquery/module_api.go index c2b5b6a2..e2c34ab7 100644 --- a/netquery/module_api.go +++ b/netquery/module_api.go @@ -87,35 +87,37 @@ func (m *module) prepare() error { } if err := api.RegisterEndpoint(api.Endpoint{ + Name: "Query Connections", + Description: "Query the in-memory sqlite connection database.", Path: "netquery/query", MimeType: "application/json", Read: api.PermitUser, // Needs read+write as the query is sent using POST data. Write: api.PermitUser, // Needs read+write as the query is sent using POST data. BelongsTo: m.Module, HandlerFunc: queryHander.ServeHTTP, - Name: "Query Connections", - Description: "Query the in-memory sqlite connection database.", }); err != nil { return fmt.Errorf("failed to register API endpoint: %w", err) } if err := api.RegisterEndpoint(api.Endpoint{ + Name: "Active Connections Chart", + Description: "Query the in-memory sqlite connection database and return a chart of active connections.", Path: "netquery/charts/connection-active", MimeType: "application/json", Write: api.PermitUser, BelongsTo: m.Module, HandlerFunc: chartHandler.ServeHTTP, - Name: "Active Connections Chart", - Description: "Query the in-memory sqlite connection database and return a chart of active connections.", }); err != nil { return fmt.Errorf("failed to register API endpoint: %w", err) } if err := api.RegisterEndpoint(api.Endpoint{ - Path: "netquery/history/clear", - MimeType: "application/json", - Write: api.PermitUser, - BelongsTo: m.Module, + Name: "Remove connections from profile history", + Description: "Remove all connections from the history database for one or more profiles", + Path: "netquery/history/clear", + MimeType: "application/json", + Write: api.PermitUser, + BelongsTo: m.Module, HandlerFunc: func(w http.ResponseWriter, r *http.Request) { var body struct { ProfileIDs []string `json:"profileIDs"` @@ -154,28 +156,21 @@ func (m *module) prepare() error { w.WriteHeader(http.StatusNoContent) }, - Name: "Remove connections from profile history", - Description: "Remove all connections from the history database for one or more profiles", }); err != nil { return fmt.Errorf("failed to register API endpoint: %w", err) - } if err := api.RegisterEndpoint(api.Endpoint{ + Name: "Apply connection history retention threshold", Path: "netquery/history/cleanup", - MimeType: "application/json", Write: api.PermitUser, BelongsTo: m.Module, - HandlerFunc: func(w http.ResponseWriter, r *http.Request) { - if err := m.Store.CleanupHistoryData(r.Context()); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - - return + ActionFunc: func(ar *api.Request) (msg string, err error) { + if err := m.Store.PurgeOldHistory(ar.Context()); err != nil { + return "", err } - - w.WriteHeader(http.StatusNoContent) + return "Deleted outdated connections.", nil }, - Name: "Apply connection history retention threshold", }); err != nil { return fmt.Errorf("failed to register API endpoint: %w", err) } @@ -184,7 +179,7 @@ func (m *module) prepare() error { } func (m *module) start() error { - m.StartServiceWorker("netquery-feeder", time.Second, func(ctx context.Context) error { + m.StartServiceWorker("netquery connection feed listener", 0, func(ctx context.Context) error { sub, err := m.db.Subscribe(query.New("network:")) if err != nil { return fmt.Errorf("failed to subscribe to network tree: %w", err) @@ -215,16 +210,12 @@ func (m *module) start() error { } }) - m.StartServiceWorker("netquery-persister", time.Second, func(ctx context.Context) error { + m.StartServiceWorker("netquery connection feed handler", 0, func(ctx context.Context) error { m.mng.HandleFeed(ctx, m.feed) return nil }) - m.StartServiceWorker("history-row-cleaner", time.Hour, func(ctx context.Context) error { - return m.Store.CleanupHistoryData(ctx) - }) - - m.StartServiceWorker("netquery-row-cleaner", time.Second, func(ctx context.Context) error { + m.StartServiceWorker("netquery live db cleaner", 0, func(ctx context.Context) error { for { select { case <-ctx.Done(): @@ -233,14 +224,18 @@ func (m *module) start() error { threshold := time.Now().Add(-network.DeleteConnsAfterEndedThreshold) count, err := m.Store.Cleanup(ctx, threshold) if err != nil { - log.Errorf("netquery: failed to count number of rows in memory: %s", err) + log.Errorf("netquery: failed to removed old connections from live db: %s", err) } else { - log.Tracef("netquery: successfully removed %d old rows that ended before %s", count, threshold) + log.Tracef("netquery: successfully removed %d old connections from live db that ended before %s", count, threshold) } } } }) + m.NewTask("network history cleaner", func(ctx context.Context, _ *modules.Task) error { + return m.Store.PurgeOldHistory(ctx) + }).Repeat(time.Hour).Schedule(time.Now().Add(10 * time.Minute)) + // For debugging, provide a simple direct SQL query interface using // the runtime database. // Only expose in development mode. @@ -269,9 +264,9 @@ func (m *module) stop() error { if err := m.mng.store.Close(); err != nil { log.Errorf("netquery: failed to close sqlite database: %s", err) } else { - // try to vaccum the history database now + // Clear deleted connections from database. if err := VacuumHistory(ctx); err != nil { - log.Errorf("netquery: failed to execute VACCUM in history database: %s", err) + log.Errorf("netquery: failed to execute VACUUM in history database: %s", err) } } diff --git a/network/connection.go b/network/connection.go index 2670390b..10d5d586 100644 --- a/network/connection.go +++ b/network/connection.go @@ -594,7 +594,7 @@ func (conn *Connection) UpdateFeatures() error { if user.MayUse(account.FeatureHistory) { lProfile := conn.Process().Profile() if lProfile != nil { - conn.HistoryEnabled = lProfile.HistoryEnabled() + conn.HistoryEnabled = lProfile.EnableHistory() } } diff --git a/profile/config.go b/profile/config.go index e9a6b963..761102ca 100644 --- a/profile/config.go +++ b/profile/config.go @@ -112,9 +112,9 @@ var ( cfgOptionEnableHistory config.BoolOption cfgOptionEnableHistoryOrder = 96 - CfgOptionHistoryRetentionKey = "history/retention" - CfgOptionHistoryRetention config.IntOption - cfgOptionHistoryRetentionOrder = 97 + CfgOptionKeepHistoryKey = "history/keep" + cfgOptionKeepHistory config.IntOption + cfgOptionKeepHistoryOrder = 97 // Setting "Enable SPN" at order 128. @@ -252,7 +252,7 @@ func registerConfiguration() error { //nolint:maintidx // Enable History err = config.Register(&config.Option{ - Name: "Enable Connection History", + Name: "Enable Network History", Key: CfgOptionEnableHistoryKey, Description: "Save connections in a database (on disk) in order to view and search them later. Changes might take a couple minutes to apply to all connections.", OptType: config.OptTypeBool, @@ -261,7 +261,7 @@ func registerConfiguration() error { //nolint:maintidx DefaultValue: false, Annotations: config.Annotations{ config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder, - config.CategoryAnnotation: "History", + config.CategoryAnnotation: "General", config.RequiresFeatureID: account.FeatureHistory, }, }) @@ -272,25 +272,29 @@ func registerConfiguration() error { //nolint:maintidx cfgBoolOptions[CfgOptionEnableHistoryKey] = cfgOptionEnableHistory err = config.Register(&config.Option{ - Name: "History Data Retention", - Key: CfgOptionHistoryRetentionKey, - Description: "How low, in days, connections should be kept in history.", + Name: "Keep Network History", + Key: CfgOptionKeepHistoryKey, + Description: `Specify how many days the network history data should be kept. Please keep in mind that more available history data makes reports (coming soon) a lot more useful. + +Older data is deleted in intervals and cleared from the database continually. If in a hurry, shutdown or restart Portmaster to clear deleted entries immediately. + +Set to 0 days to keep network history forever. Depending on your device, this might affect performance.`, OptType: config.OptTypeInt, ReleaseLevel: config.ReleaseLevelStable, ExpertiseLevel: config.ExpertiseLevelUser, - DefaultValue: 7, + DefaultValue: 30, Annotations: config.Annotations{ config.UnitAnnotation: "Days", - config.DisplayOrderAnnotation: cfgOptionHistoryRetentionOrder, - config.CategoryAnnotation: "History", + config.DisplayOrderAnnotation: cfgOptionKeepHistoryOrder, + config.CategoryAnnotation: "General", config.RequiresFeatureID: account.FeatureHistory, }, }) if err != nil { return err } - CfgOptionHistoryRetention = config.Concurrent.GetAsInt(CfgOptionHistoryRetentionKey, 7) - cfgIntOptions[CfgOptionHistoryRetentionKey] = CfgOptionHistoryRetention + cfgOptionKeepHistory = config.Concurrent.GetAsInt(CfgOptionKeepHistoryKey, 30) + cfgIntOptions[CfgOptionKeepHistoryKey] = cfgOptionKeepHistory rulesHelp := strings.ReplaceAll(`Rules are checked from top to bottom, stopping after the first match. They can match: