diff --git a/database/database_test.go b/database/database_test.go index 6367648..8d20a0f 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -1,6 +1,7 @@ package database import ( + "context" "fmt" "io/ioutil" "log" @@ -162,7 +163,7 @@ func TestDatabaseSystem(t *testing.T) { testDatabase(t, "fstree") testDatabase(t, "hashmap") - err = MaintainRecordStates() + err = MaintainRecordStates(context.TODO()) if err != nil { t.Fatal(err) } diff --git a/database/dbmodule/maintenance.go b/database/dbmodule/maintenance.go index 6a1755c..0d2a42b 100644 --- a/database/dbmodule/maintenance.go +++ b/database/dbmodule/maintenance.go @@ -5,6 +5,7 @@ import ( "time" "github.com/safing/portbase/database" + "github.com/safing/portbase/log" "github.com/safing/portbase/modules" ) @@ -15,13 +16,16 @@ func startMaintenanceTasks() { } func maintainBasic(ctx context.Context, task *modules.Task) error { + log.Infof("database: running Maintain") return database.Maintain() } func maintainThorough(ctx context.Context, task *modules.Task) error { + log.Infof("database: running MaintainThorough") return database.MaintainThorough() } func maintainRecords(ctx context.Context, task *modules.Task) error { - return database.MaintainRecordStates() + log.Infof("database: running MaintainRecordStates") + return database.MaintainRecordStates(ctx) } diff --git a/database/maintenance.go b/database/maintenance.go index 09c0fa7..1f0e907 100644 --- a/database/maintenance.go +++ b/database/maintenance.go @@ -1,16 +1,21 @@ package database import ( + "context" "time" + "github.com/tevino/abool" + "github.com/safing/portbase/database/query" "github.com/safing/portbase/database/record" ) // Maintain runs the Maintain method on all storages. func Maintain() (err error) { - controllers := duplicateControllers() - for _, c := range controllers { + // copy, as we might use the very long + all := duplicateControllers() + + for _, c := range all { err = c.Maintain() if err != nil { return @@ -21,7 +26,9 @@ func Maintain() (err error) { // MaintainThorough runs the MaintainThorough method on all storages. func MaintainThorough() (err error) { + // copy, as we might use the very long all := duplicateControllers() + for _, c := range all { err = c.MaintainThorough() if err != nil { @@ -32,12 +39,32 @@ func MaintainThorough() (err error) { } // MaintainRecordStates runs record state lifecycle maintenance on all storages. -func MaintainRecordStates() error { +func MaintainRecordStates(ctx context.Context) error { //nolint:gocognit + // TODO: Put this in the storage interface to correctly maintain on all storages. + // Storages might check for deletion and expiry in the query interface and not return anything here. + + // listen for ctx cancel + stop := abool.New() + doneCh := make(chan struct{}) // for goroutine cleanup + defer close(doneCh) + go func() { + select { + case <-ctx.Done(): + case <-doneCh: + } + stop.Set() + }() + + // copy, as we might use the very long all := duplicateControllers() + now := time.Now().Unix() thirtyDaysAgo := time.Now().Add(-30 * 24 * time.Hour).Unix() for _, c := range all { + if stop.IsSet() { + return nil + } if c.ReadOnly() || c.Injected() { continue @@ -56,30 +83,52 @@ func MaintainRecordStates() error { var toDelete []record.Record var toExpire []record.Record - for r := range it.Next { - switch { - case r.Meta().Deleted < thirtyDaysAgo: - toDelete = append(toDelete, r) - case r.Meta().Expires < now: - toExpire = append(toExpire, r) + queryLoop: + for { + select { + case r := <-it.Next: + if r == nil { + break queryLoop + } + + meta := r.Meta() + switch { + case meta.Deleted > 0 && meta.Deleted < thirtyDaysAgo: + toDelete = append(toDelete, r) + case meta.Expires > 0 && meta.Expires < now: + toExpire = append(toExpire, r) + } + case <-ctx.Done(): + it.Cancel() + break queryLoop } } if it.Err() != nil { return err } + if stop.IsSet() { + return nil + } for _, r := range toDelete { err := c.storage.Delete(r.DatabaseKey()) if err != nil { return err } + if stop.IsSet() { + return nil + } } + for _, r := range toExpire { r.Meta().Delete() err := c.Put(r) if err != nil { return err } + if stop.IsSet() { + return nil + } } } @@ -87,9 +136,10 @@ func MaintainRecordStates() error { } func duplicateControllers() (all []*Controller) { - controllersLock.Lock() - defer controllersLock.Unlock() + controllersLock.RLock() + defer controllersLock.RUnlock() + all = make([]*Controller, len(controllers)) for _, c := range controllers { all = append(all, c) }