Move DB record maintenance to storage interface

database/storage/badger/badger.go

@@ -1,6 +1,7 @@
 package badger
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"time"
@@ -193,19 +194,25 @@ func (b *Badger) Injected() bool {
 }
 
 // Maintain runs a light maintenance operation on the database.
-func (b *Badger) Maintain() error {
+func (b *Badger) Maintain(_ context.Context) error {
 	_ = b.db.RunValueLogGC(0.7)
 	return nil
 }
 
 // MaintainThorough runs a thorough maintenance operation on the database.
-func (b *Badger) MaintainThorough() (err error) {
+func (b *Badger) MaintainThorough(_ context.Context) (err error) {
 	for err == nil {
 		err = b.db.RunValueLogGC(0.7)
 	}
 	return nil
 }
 
+// MaintainRecordStates maintains records states in the database.
+func (b *Badger) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
+	// TODO: implement MaintainRecordStates
+	return nil
+}
+
 // Shutdown shuts down the database.
 func (b *Badger) Shutdown() error {
 	return b.db.Close()

database/storage/badger/badger_test.go

@@ -2,6 +2,7 @@
 package badger
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -116,11 +117,11 @@ func TestBadger(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.Maintain()
+	err = db.Maintain(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = db.MaintainThorough()
+	err = db.MaintainThorough(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}

database/storage/bbolt/bbolt.go

@@ -2,6 +2,7 @@ package bbolt
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -235,15 +236,69 @@ func (b *BBolt) Injected() bool {
 }
 
 // Maintain runs a light maintenance operation on the database.
-func (b *BBolt) Maintain() error {
+func (b *BBolt) Maintain(_ context.Context) error {
 	return nil
 }
 
 // MaintainThorough runs a thorough maintenance operation on the database.
-func (b *BBolt) MaintainThorough() (err error) {
+func (b *BBolt) MaintainThorough(_ context.Context) error {
 	return nil
 }
 
+// MaintainRecordStates maintains records states in the database.
+func (b *BBolt) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
+	now := time.Now().Unix()
+	purgeThreshold := purgeDeletedBefore.Unix()
+
+	return b.db.Update(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(bucketName)
+		// Create a cursor for iteration.
+		c := bucket.Cursor()
+		for key, value := c.First(); key != nil; key, value = c.Next() {
+			// wrap value
+			wrapper, err := record.NewRawWrapper(b.name, string(key), value)
+			if err != nil {
+				return err
+			}
+
+			// check if we need to do maintenance
+			meta := wrapper.Meta()
+			switch {
+			case meta.Deleted > 0 && meta.Deleted < purgeThreshold:
+				// delete from storage
+				err = c.Delete()
+				if err != nil {
+					return err
+				}
+			case meta.Expires > 0 && meta.Expires < now:
+				// mark as deleted
+				meta.Deleted = meta.Expires
+				deleted, err := wrapper.MarshalRecord(wrapper)
+				if err != nil {
+					return err
+				}
+				err = bucket.Put(key, deleted)
+				if err != nil {
+					return err
+				}
+
+				// reposition cursor
+				c.Seek(key)
+			}
+
+			time.Sleep(100 * time.Millisecond)
+
+			// check if context is cancelled
+			select {
+			case <-ctx.Done():
+				return nil
+			default:
+			}
+		}
+		return nil
+	})
+}
+
 // Shutdown shuts down the database.
 func (b *BBolt) Shutdown() error {
 	return b.db.Close()

database/storage/bbolt/bbolt_test.go

@@ -2,11 +2,13 @@
 package bbolt
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"reflect"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
@@ -144,11 +146,15 @@ func TestBBolt(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.Maintain()
+	err = db.Maintain(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = db.MaintainThorough()
+	err = db.MaintainThorough(context.TODO())
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.MaintainRecordStates(context.TODO(), time.Now())
 	if err != nil {
 		t.Fatal(err)
 	}

database/storage/fstree/fstree.go

@@ -5,6 +5,7 @@ It is primarily meant for easy testing or storing big files that can easily be a
 package fstree
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -255,12 +256,18 @@ func (fst *FSTree) Injected() bool {
 }
 
 // Maintain runs a light maintenance operation on the database.
-func (fst *FSTree) Maintain() error {
+func (fst *FSTree) Maintain(_ context.Context) error {
 	return nil
 }
 
 // MaintainThorough runs a thorough maintenance operation on the database.
-func (fst *FSTree) MaintainThorough() error {
+func (fst *FSTree) MaintainThorough(_ context.Context) error {
+	return nil
+}
+
+// MaintainRecordStates maintains records states in the database.
+func (fst *FSTree) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
+	// TODO: implement MaintainRecordStates
 	return nil
 }
 

database/storage/hashmap/map.go

@@ -1,6 +1,7 @@
 package hashmap
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sync"
@@ -146,12 +147,44 @@ func (hm *HashMap) Injected() bool {
 }
 
 // Maintain runs a light maintenance operation on the database.
-func (hm *HashMap) Maintain() error {
+func (hm *HashMap) Maintain(_ context.Context) error {
 	return nil
 }
 
 // MaintainThorough runs a thorough maintenance operation on the database.
-func (hm *HashMap) MaintainThorough() (err error) {
+func (hm *HashMap) MaintainThorough(_ context.Context) error {
+	return nil
+}
+
+// MaintainRecordStates maintains records states in the database.
+func (hm *HashMap) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
+	hm.dbLock.Lock()
+	defer hm.dbLock.Unlock()
+
+	now := time.Now().Unix()
+	purgeThreshold := purgeDeletedBefore.Unix()
+
+	for key, record := range hm.db {
+		meta := record.Meta()
+		switch {
+		case meta.Deleted > 0 && meta.Deleted < purgeThreshold:
+			// delete from storage
+			delete(hm.db, key)
+		case meta.Expires > 0 && meta.Expires < now:
+			// mark as deleted
+			record.Lock()
+			meta.Deleted = meta.Expires
+			record.Unlock()
+		}
+
+		// check if context is cancelled
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+		}
+	}
+
 	return nil
 }
 

database/storage/hashmap/map_test.go

@@ -2,6 +2,7 @@
 package hashmap
 
 import (
+	"context"
 	"reflect"
 	"sync"
 	"testing"
@@ -130,11 +131,11 @@ func TestHashMap(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.Maintain()
+	err = db.Maintain(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = db.MaintainThorough()
+	err = db.MaintainThorough(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}

database/storage/injectbase.go

@@ -1,7 +1,9 @@
 package storage
 
 import (
+	"context"
 	"errors"
+	"time"
 
 	"github.com/safing/portbase/database/iterator"
 	"github.com/safing/portbase/database/query"
@@ -54,12 +56,17 @@ func (i *InjectBase) Injected() bool {
 }
 
 // Maintain runs a light maintenance operation on the database.
-func (i *InjectBase) Maintain() error {
+func (i *InjectBase) Maintain(ctx context.Context) error {
 	return nil
 }
 
 // MaintainThorough runs a thorough maintenance operation on the database.
-func (i *InjectBase) MaintainThorough() error {
+func (i *InjectBase) MaintainThorough(ctx context.Context) error {
+	return nil
+}
+
+// MaintainRecordStates maintains records states in the database.
+func (i *InjectBase) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	return nil
 }
 

database/storage/interface.go

@@ -1,6 +1,9 @@
 package storage
 
 import (
+	"context"
+	"time"
+
 	"github.com/safing/portbase/database/iterator"
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
@@ -15,8 +18,9 @@ type Interface interface {
 
 	ReadOnly() bool
 	Injected() bool
-	Maintain() error
-	MaintainThorough() error
+	Maintain(ctx context.Context) error
+	MaintainThorough(ctx context.Context) error
+	MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error
 	Shutdown() error
 }
 

database/storage/sinkhole/sinkhole.go

@@ -1,7 +1,9 @@
 package sinkhole
 
 import (
+	"context"
 	"errors"
+	"time"
 
 	"github.com/safing/portbase/database/iterator"
 	"github.com/safing/portbase/database/query"
@@ -77,12 +79,17 @@ func (s *Sinkhole) Injected() bool {
 }
 
 // Maintain runs a light maintenance operation on the database.
-func (s *Sinkhole) Maintain() error {
+func (s *Sinkhole) Maintain(ctx context.Context) error {
 	return nil
 }
 
 // MaintainThorough runs a thorough maintenance operation on the database.
-func (s *Sinkhole) MaintainThorough() (err error) {
+func (s *Sinkhole) MaintainThorough(ctx context.Context) error {
+	return nil
+}
+
+// MaintainRecordStates maintains records states in the database.
+func (s *Sinkhole) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	return nil
 }