vikarti.anatra/safing-portmaster
1
0
Fork 0
mirror of https://github.com/safing/portmaster synced 2025-09-02 02:29:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Improve history purging

Browse source
This commit is contained in:
Daniel 2023-08-09 14:45:56 +02:00
parent a722b27c01
commit cf70c55ab5
5 changed files with 101 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
netquery/database.go
View file

@ -16,6 +16,7 @@ import (
"zombiezen.com/go/sqlite"
"zombiezen.com/go/sqlite/sqlitex"
"github.com/safing/portbase/config"
"github.com/safing/portbase/dataroot"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/netquery/orm"
@ -203,6 +204,7 @@ func NewInMemory() (*Database, error) {
return db, nil
}
// Close closes the database, including pools and connections.
func (db *Database) Close() error {
db.readConnPool.Close()
@ -213,7 +215,8 @@ func (db *Database) Close() error {
return nil
}
func VacuumHistory(ctx context.Context) error {
// VacuumHistory rewrites the history database in order to purge deleted records.
func VacuumHistory(ctx context.Context) (err error) {
historyParentDir := dataroot.Root().ChildDir("databases", 0o700)
if err := historyParentDir.Ensure(); err != nil {
return fmt.Errorf("failed to ensure database directory exists: %w", err)
@ -235,6 +238,11 @@ func VacuumHistory(ctx context.Context) error {
if err != nil {
return err
}
defer func() {
if closeErr := writeConn.Close(); closeErr != nil && err == nil {
err = closeErr
}
}()
return orm.RunQuery(ctx, writeConn, "VACUUM")
}
@ -414,50 +422,72 @@ func (db *Database) dumpTo(ctx context.Context, w io.Writer) error { //nolint:un
return enc.Encode(conns)
}
func (db *Database) CleanupHistoryData(ctx context.Context) error {
query := "SELECT DISTINCT profile FROM history.connections"
// PurgeOldHistory deletes history data outside of the (per-app) retention time frame.
func (db *Database) PurgeOldHistory(ctx context.Context) error {
// Setup tracer for the clean up process.
ctx, tracer := log.AddTracer(ctx)
defer tracer.Submit()
defer tracer.Info("history: deleted connections outside of retention from %d profiles")
// Get list of profiles in history.
query := "SELECT DISTINCT profile FROM history.connections"
var result []struct {
Profile string `sqlite:"profile"`
}
if err := db.Execute(ctx, query, orm.WithResult(&result)); err != nil {
return fmt.Errorf("failed to get a list of profiles from the history database: %w", err)
}
globalRetentionDays := profile.CfgOptionHistoryRetention()
merr := new(multierror.Error)
var (
// Get global retention days - do not delete in case of error.
globalRetentionDays = config.GetAsInt(profile.CfgOptionKeepHistoryKey, 0)()
profileName string
retentionDays int64
profileCnt int
merr = new(multierror.Error)
)
for _, row := range result {
// Get profile and retention days.
id := strings.TrimPrefix(row.Profile, string(profile.SourceLocal)+"/")
p, err := profile.GetLocalProfile(id, nil, nil)
var retention int
if err == nil {
retention = p.HistoryRetention()
profileName = p.String()
retentionDays = p.LayeredProfile().KeepHistory()
} else {
// we failed to get the profile, fallback to the global setting
log.Errorf("failed to load profile for id %s: %s", id, err)
retention = int(globalRetentionDays)
// Getting profile failed, fallback to global setting.
tracer.Errorf("history: failed to load profile for id %s: %s", id, err)
profileName = row.Profile
retentionDays = globalRetentionDays
}
if retention == 0 {
log.Infof("skipping history data retention for profile %s: retention is disabled", row.Profile)
// Skip deleting if history should be kept forever.
if retentionDays == 0 {
tracer.Tracef("history: retention is disabled for %s, skipping", profileName)
continue
}
// Count profiles where connections were deleted.
profileCnt++
threshold := time.Now().Add(-1 * time.Duration(retention) * time.Hour * 24)
log.Infof("cleaning up history data for profile %s with retention setting %d days (threshold = %s)", row.Profile, retention, threshold.Format(orm.SqliteTimeFormat))
query := "DELETE FROM history.connections WHERE profile = :profile AND active = FALSE AND datetime(started) < datetime(:threshold)"
if err := db.ExecuteWrite(ctx, query, orm.WithNamedArgs(map[string]any{
":profile": row.Profile,
":threshold": threshold.Format(orm.SqliteTimeFormat),
})); err != nil {
log.Errorf("failed to delete connections for profile %s from history: %s", row.Profile, err)
// TODO: count cleared connections
threshold := time.Now().Add(-1 * time.Duration(retentionDays) * time.Hour * 24)
if err := db.ExecuteWrite(ctx,
"DELETE FROM history.connections WHERE profile = :profile AND active = FALSE AND datetime(started) < datetime(:threshold)",
orm.WithNamedArgs(map[string]any{
":profile": row.Profile,
":threshold": threshold.Format(orm.SqliteTimeFormat),
}),
); err != nil {
tracer.Warningf("history: failed to delete connections of %s: %s", profileName, err)
merr.Errors = append(merr.Errors, fmt.Errorf("profile %s: %w", row.Profile, err))
} else {
tracer.Debugf(
"history: deleted connections older than %d days (before %s) of %s",
retentionDays,
threshold,
profileName,
)
}
}

4
netquery/manager.go
View file

@ -40,8 +40,8 @@ type (
// the bandwidth data to the history database.
UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, bytesReceived uint64, bytesSent uint64) error
// CleanupHistoryData applies data retention to the history database.
CleanupHistoryData(ctx context.Context) error
// PurgeOldHistory deletes data outside of the retention time frame from the history database.
PurgeOldHistory(ctx context.Context) error
// Close closes the connection store. It must not be used afterwards.
Close() error

57
netquery/module_api.go
View file

@ -87,35 +87,37 @@ func (m *module) prepare() error {
}
if err := api.RegisterEndpoint(api.Endpoint{
Name: "Query Connections",
Description: "Query the in-memory sqlite connection database.",
Path: "netquery/query",
MimeType: "application/json",
Read: api.PermitUser, // Needs read+write as the query is sent using POST data.
Write: api.PermitUser, // Needs read+write as the query is sent using POST data.
BelongsTo: m.Module,
HandlerFunc: queryHander.ServeHTTP,
Name: "Query Connections",
Description: "Query the in-memory sqlite connection database.",
}); err != nil {
return fmt.Errorf("failed to register API endpoint: %w", err)
}
if err := api.RegisterEndpoint(api.Endpoint{
Name: "Active Connections Chart",
Description: "Query the in-memory sqlite connection database and return a chart of active connections.",
Path: "netquery/charts/connection-active",
MimeType: "application/json",
Write: api.PermitUser,
BelongsTo: m.Module,
HandlerFunc: chartHandler.ServeHTTP,
Name: "Active Connections Chart",
Description: "Query the in-memory sqlite connection database and return a chart of active connections.",
}); err != nil {
return fmt.Errorf("failed to register API endpoint: %w", err)
}
if err := api.RegisterEndpoint(api.Endpoint{
Path: "netquery/history/clear",
MimeType: "application/json",
Write: api.PermitUser,
BelongsTo: m.Module,
Name: "Remove connections from profile history",
Description: "Remove all connections from the history database for one or more profiles",
Path: "netquery/history/clear",
MimeType: "application/json",
Write: api.PermitUser,
BelongsTo: m.Module,
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
var body struct {
ProfileIDs []string `json:"profileIDs"`
@ -154,28 +156,21 @@ func (m *module) prepare() error {
w.WriteHeader(http.StatusNoContent)
},
Name: "Remove connections from profile history",
Description: "Remove all connections from the history database for one or more profiles",
}); err != nil {
return fmt.Errorf("failed to register API endpoint: %w", err)
}
if err := api.RegisterEndpoint(api.Endpoint{
Name: "Apply connection history retention threshold",
Path: "netquery/history/cleanup",
MimeType: "application/json",
Write: api.PermitUser,
BelongsTo: m.Module,
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
if err := m.Store.CleanupHistoryData(r.Context()); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
ActionFunc: func(ar *api.Request) (msg string, err error) {
if err := m.Store.PurgeOldHistory(ar.Context()); err != nil {
return "", err
}
w.WriteHeader(http.StatusNoContent)
return "Deleted outdated connections.", nil
},
Name: "Apply connection history retention threshold",
}); err != nil {
return fmt.Errorf("failed to register API endpoint: %w", err)
}
@ -184,7 +179,7 @@ func (m *module) prepare() error {
}
func (m *module) start() error {
m.StartServiceWorker("netquery-feeder", time.Second, func(ctx context.Context) error {
m.StartServiceWorker("netquery connection feed listener", 0, func(ctx context.Context) error {
sub, err := m.db.Subscribe(query.New("network:"))
if err != nil {
return fmt.Errorf("failed to subscribe to network tree: %w", err)
@ -215,16 +210,12 @@ func (m *module) start() error {
}
})
m.StartServiceWorker("netquery-persister", time.Second, func(ctx context.Context) error {
m.StartServiceWorker("netquery connection feed handler", 0, func(ctx context.Context) error {
m.mng.HandleFeed(ctx, m.feed)
return nil
})
m.StartServiceWorker("history-row-cleaner", time.Hour, func(ctx context.Context) error {
return m.Store.CleanupHistoryData(ctx)
})
m.StartServiceWorker("netquery-row-cleaner", time.Second, func(ctx context.Context) error {
m.StartServiceWorker("netquery live db cleaner", 0, func(ctx context.Context) error {
for {
select {
case <-ctx.Done():
@ -233,14 +224,18 @@ func (m *module) start() error {
threshold := time.Now().Add(-network.DeleteConnsAfterEndedThreshold)
count, err := m.Store.Cleanup(ctx, threshold)
if err != nil {
log.Errorf("netquery: failed to count number of rows in memory: %s", err)
log.Errorf("netquery: failed to removed old connections from live db: %s", err)
} else {
log.Tracef("netquery: successfully removed %d old rows that ended before %s", count, threshold)
log.Tracef("netquery: successfully removed %d old connections from live db that ended before %s", count, threshold)
}
}
}
})
m.NewTask("network history cleaner", func(ctx context.Context, _ *modules.Task) error {
return m.Store.PurgeOldHistory(ctx)
}).Repeat(time.Hour).Schedule(time.Now().Add(10 * time.Minute))
// For debugging, provide a simple direct SQL query interface using
// the runtime database.
// Only expose in development mode.
@ -269,9 +264,9 @@ func (m *module) stop() error {
if err := m.mng.store.Close(); err != nil {
log.Errorf("netquery: failed to close sqlite database: %s", err)
} else {
// try to vaccum the history database now
// Clear deleted connections from database.
if err := VacuumHistory(ctx); err != nil {
log.Errorf("netquery: failed to execute VACCUM in history database: %s", err)
log.Errorf("netquery: failed to execute VACUUM in history database: %s", err)
}
}

2
network/connection.go
View file

@ -594,7 +594,7 @@ func (conn *Connection) UpdateFeatures() error {
if user.MayUse(account.FeatureHistory) {
lProfile := conn.Process().Profile()
if lProfile != nil {
conn.HistoryEnabled = lProfile.HistoryEnabled()
conn.HistoryEnabled = lProfile.EnableHistory()
}
}

30
profile/config.go
View file

@ -112,9 +112,9 @@ var (
cfgOptionEnableHistory config.BoolOption
cfgOptionEnableHistoryOrder = 96
CfgOptionHistoryRetentionKey = "history/retention"
CfgOptionHistoryRetention config.IntOption
cfgOptionHistoryRetentionOrder = 97
CfgOptionKeepHistoryKey = "history/keep"
cfgOptionKeepHistory config.IntOption
cfgOptionKeepHistoryOrder = 97
// Setting "Enable SPN" at order 128.
@ -252,7 +252,7 @@ func registerConfiguration() error { //nolint:maintidx
// Enable History
err = config.Register(&config.Option{
Name: "Enable Connection History",
Name: "Enable Network History",
Key: CfgOptionEnableHistoryKey,
Description: "Save connections in a database (on disk) in order to view and search them later. Changes might take a couple minutes to apply to all connections.",
OptType: config.OptTypeBool,
@ -261,7 +261,7 @@ func registerConfiguration() error { //nolint:maintidx
DefaultValue: false,
Annotations: config.Annotations{
config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder,
config.CategoryAnnotation: "History",
config.CategoryAnnotation: "General",
config.RequiresFeatureID: account.FeatureHistory,
},
})
@ -272,25 +272,29 @@ func registerConfiguration() error { //nolint:maintidx
cfgBoolOptions[CfgOptionEnableHistoryKey] = cfgOptionEnableHistory
err = config.Register(&config.Option{
Name: "History Data Retention",
Key: CfgOptionHistoryRetentionKey,
Description: "How low, in days, connections should be kept in history.",
Name: "Keep Network History",
Key: CfgOptionKeepHistoryKey,
Description: `Specify how many days the network history data should be kept. Please keep in mind that more available history data makes reports (coming soon) a lot more useful.
Older data is deleted in intervals and cleared from the database continually. If in a hurry, shutdown or restart Portmaster to clear deleted entries immediately.
Set to 0 days to keep network history forever. Depending on your device, this might affect performance.`,
OptType: config.OptTypeInt,
ReleaseLevel: config.ReleaseLevelStable,
ExpertiseLevel: config.ExpertiseLevelUser,
DefaultValue: 7,
DefaultValue: 30,
Annotations: config.Annotations{
config.UnitAnnotation: "Days",
config.DisplayOrderAnnotation: cfgOptionHistoryRetentionOrder,
config.CategoryAnnotation: "History",
config.DisplayOrderAnnotation: cfgOptionKeepHistoryOrder,
config.CategoryAnnotation: "General",
config.RequiresFeatureID: account.FeatureHistory,
},
})
if err != nil {
return err
}
CfgOptionHistoryRetention = config.Concurrent.GetAsInt(CfgOptionHistoryRetentionKey, 7)
cfgIntOptions[CfgOptionHistoryRetentionKey] = CfgOptionHistoryRetention
cfgOptionKeepHistory = config.Concurrent.GetAsInt(CfgOptionKeepHistoryKey, 30)
cfgIntOptions[CfgOptionKeepHistoryKey] = cfgOptionKeepHistory
rulesHelp := strings.ReplaceAll(`Rules are checked from top to bottom, stopping after the first match. They can match: