Merge pull request #82 from safing/feature/deletemany

Make shadow deletes conditional

config/database.go

@@ -134,7 +134,6 @@ func registerAsDatabase() error {
 		Name:        "config",
 		Description: "Configuration Manager",
 		StorageType: "injected",
-		PrimaryAPI:  "",
 	})
 	if err != nil {
 		return err

database/controller.go

@@ -17,6 +17,7 @@ import (
 // A Controller takes care of all the extra database logic.
 type Controller struct {
 	storage      storage.Interface
+	shadowDelete bool
 
 	hooks         []*RegisteredHook
 	subscriptions []*Subscription
@@ -33,9 +34,10 @@ type Controller struct {
 }
 
 // newController creates a new controller for a storage.
-func newController(storageInt storage.Interface) *Controller {
+func newController(storageInt storage.Interface, shadowDelete bool) *Controller {
 	return &Controller{
 		storage:      storageInt,
+		shadowDelete: shadowDelete,
 		migrating:    abool.NewBool(false),
 		hibernating:  abool.NewBool(false),
 	}
@@ -122,6 +124,14 @@ func (c *Controller) Put(r record.Record) (err error) {
 		}
 	}
 
+	if !c.shadowDelete && r.Meta().IsDeleted() {
+		// Immediate delete.
+		err = c.storage.Delete(r.DatabaseKey())
+		if err != nil {
+			return err
+		}
+	} else {
+		// Put or shadow delete.
 		r, err = c.storage.Put(r)
 		if err != nil {
 			return err
@@ -129,6 +139,7 @@ func (c *Controller) Put(r record.Record) (err error) {
 		if r == nil {
 			return errors.New("storage returned nil record after successful put operation")
 		}
+	}
 
 	// process subscriptions
 	for _, sub := range c.subscriptions {
@@ -161,7 +172,7 @@ func (c *Controller) PutMany() (chan<- record.Record, <-chan error) {
 	}
 
 	if batcher, ok := c.storage.(storage.Batcher); ok {
-		return batcher.PutMany()
+		return batcher.PutMany(c.shadowDelete)
 	}
 
 	errs := make(chan error, 1)
@@ -236,7 +247,10 @@ func (c *Controller) Maintain(ctx context.Context) error {
 		return nil
 	}
 
-	return c.storage.Maintain(ctx)
+	if maintainer, ok := c.storage.(storage.Maintainer); ok {
+		return maintainer.Maintain(ctx)
+	}
+	return nil
 }
 
 // MaintainThorough runs the MaintainThorough method on the storage.
@@ -248,7 +262,10 @@ func (c *Controller) MaintainThorough(ctx context.Context) error {
 		return nil
 	}
 
-	return c.storage.MaintainThorough(ctx)
+	if maintainer, ok := c.storage.(storage.Maintainer); ok {
+		return maintainer.MaintainThorough(ctx)
+	}
+	return nil
 }
 
 // MaintainRecordStates runs the record state lifecycle maintenance on the storage.

database/controllers.go

@@ -51,7 +51,7 @@ func getController(name string) (*Controller, error) {
 		return nil, fmt.Errorf(`could not start database %s (type %s): %s`, name, registeredDB.StorageType, err)
 	}
 
-	controller = newController(storageInt)
+	controller = newController(storageInt, registeredDB.ShadowDelete)
 	controllers[name] = controller
 	return controller, nil
 }
@@ -82,7 +82,7 @@ func InjectDatabase(name string, storageInt storage.Interface) (*Controller, err
 		return nil, fmt.Errorf(`database not of type "injected"`)
 	}
 
-	controller := newController(storageInt)
+	controller := newController(storageInt, false)
 	controllers[name] = controller
 	return controller, nil
 }

database/database.go

@@ -10,7 +10,7 @@ type Database struct {
 	Name         string
 	Description  string
 	StorageType  string
-	PrimaryAPI  string
+	ShadowDelete bool // Whether deleted records should be kept until purged.
 	Registered   time.Time
 	LastUpdated  time.Time
 	LastLoaded   time.Time

database/database_test.go

@@ -34,7 +34,7 @@ func testDatabase(t *testing.T, storageType string, testPutMany, testRecordMaint
 			Name:         dbName,
 			Description:  fmt.Sprintf("Unit Test Database for %s", storageType),
 			StorageType:  storageType,
-			PrimaryAPI:  "",
+			ShadowDelete: true,
 		})
 		if err != nil {
 			t.Fatal(err)

database/registry.go

@@ -49,8 +49,8 @@ func Register(new *Database) (*Database, error) {
 			registeredDB.Description = new.Description
 			save = true
 		}
-		if registeredDB.PrimaryAPI != new.PrimaryAPI {
+		if registeredDB.ShadowDelete != new.ShadowDelete {
-			registeredDB.PrimaryAPI = new.PrimaryAPI
+			registeredDB.ShadowDelete = new.ShadowDelete
 			save = true
 		}
 	} else {

database/storage/badger/badger_test.go

@@ -11,6 +11,13 @@ import (
 
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/database/storage"
+)
+
+var (
+	// Compile time interface checks.
+	_ storage.Interface  = &Badger{}
+	_ storage.Maintainer = &Badger{}
 )
 
 type TestRecord struct {
@@ -117,14 +124,19 @@ func TestBadger(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.Maintain(context.TODO())
+	maintainer, ok := db.(storage.Maintainer)
+	if ok {
+		err = maintainer.Maintain(context.TODO())
 		if err != nil {
 			t.Fatal(err)
 		}
-	err = db.MaintainThorough(context.TODO())
+		err = maintainer.MaintainThorough(context.TODO())
 		if err != nil {
 			t.Fatal(err)
 		}
+	} else {
+		t.Fatal("should implement Maintainer")
+	}
 
 	// shutdown
 	err = db.Shutdown()

database/storage/bbolt/bbolt.go

@@ -107,7 +107,7 @@ func (b *BBolt) Put(r record.Record) (record.Record, error) {
 }
 
 // PutMany stores many records in the database.
-func (b *BBolt) PutMany() (chan<- record.Record, <-chan error) {
+func (b *BBolt) PutMany(shadowDelete bool) (chan<- record.Record, <-chan error) {
 	batch := make(chan record.Record, 100)
 	errs := make(chan error, 1)
 
@@ -115,6 +115,15 @@ func (b *BBolt) PutMany() (chan<- record.Record, <-chan error) {
 		err := b.db.Batch(func(tx *bbolt.Tx) error {
 			bucket := tx.Bucket(bucketName)
 			for r := range batch {
+				if !shadowDelete && r.Meta().IsDeleted() {
+					// Immediate delete.
+					txErr := bucket.Delete([]byte(r.DatabaseKey()))
+					if txErr != nil {
+						return txErr
+					}
+				} else {
+					// Put or shadow delete.
+
 					// marshal
 					data, txErr := r.MarshalRecord(r)
 					if txErr != nil {
@@ -127,6 +136,7 @@ func (b *BBolt) PutMany() (chan<- record.Record, <-chan error) {
 						return txErr
 					}
 				}
+			}
 			return nil
 		})
 		errs <- err
@@ -235,16 +245,6 @@ func (b *BBolt) Injected() bool {
 	return false
 }
 
-// Maintain runs a light maintenance operation on the database.
-func (b *BBolt) Maintain(_ context.Context) error {
-	return nil
-}
-
-// MaintainThorough runs a thorough maintenance operation on the database.
-func (b *BBolt) MaintainThorough(_ context.Context) error {
-	return nil
-}
-
 // MaintainRecordStates maintains records states in the database.
 func (b *BBolt) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	now := time.Now().Unix()

database/storage/bbolt/bbolt_test.go

@@ -12,6 +12,13 @@ import (
 
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/database/storage"
+)
+
+var (
+	// Compile time interface checks.
+	_ storage.Interface = &BBolt{}
+	_ storage.Batcher   = &BBolt{}
 )
 
 type TestRecord struct {
@@ -146,14 +153,6 @@ func TestBBolt(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.Maintain(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db.MaintainThorough(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
 	err = db.MaintainRecordStates(context.TODO(), time.Now())
 	if err != nil {
 		t.Fatal(err)

database/storage/fstree/fstree.go

@@ -255,16 +255,6 @@ func (fst *FSTree) Injected() bool {
 	return false
 }
 
-// Maintain runs a light maintenance operation on the database.
-func (fst *FSTree) Maintain(_ context.Context) error {
-	return nil
-}
-
-// MaintainThorough runs a thorough maintenance operation on the database.
-func (fst *FSTree) MaintainThorough(_ context.Context) error {
-	return nil
-}
-
 // MaintainRecordStates maintains records states in the database.
 func (fst *FSTree) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	// TODO: implement MaintainRecordStates
@@ -279,6 +269,7 @@ func (fst *FSTree) Shutdown() error {
 // writeFile mirrors ioutil.WriteFile, replacing an existing file with the same
 // name atomically. This is not atomic on Windows, but still an improvement.
 // TODO: Replace with github.com/google/renamio.WriteFile as soon as it is fixed on Windows.
+// TODO: This has become a wont-fix. Explore other options.
 // This function is forked from https://github.com/google/renameio/blob/a368f9987532a68a3d676566141654a81aa8100b/writefile.go.
 func writeFile(filename string, data []byte, perm os.FileMode) error {
 	t, err := renameio.TempFile("", filename)

database/storage/fstree/fstree_test.go

@@ -0,0 +1,8 @@
+package fstree
+
+import "github.com/safing/portbase/database/storage"
+
+var (
+	// Compile time interface checks.
+	_ storage.Interface = &FSTree{}
+)

database/storage/hashmap/map.go

@@ -54,7 +54,7 @@ func (hm *HashMap) Put(r record.Record) (record.Record, error) {
 }
 
 // PutMany stores many records in the database.
-func (hm *HashMap) PutMany() (chan<- record.Record, <-chan error) {
+func (hm *HashMap) PutMany(shadowDelete bool) (chan<- record.Record, <-chan error) {
 	hm.dbLock.Lock()
 	defer hm.dbLock.Unlock()
 	// we could lock for every record, but we want to have the same behaviour
@@ -66,8 +66,12 @@ func (hm *HashMap) PutMany() (chan<- record.Record, <-chan error) {
 	// start handler
 	go func() {
 		for r := range batch {
+			if !shadowDelete && r.Meta().IsDeleted() {
+				delete(hm.db, r.DatabaseKey())
+			} else {
 				hm.db[r.DatabaseKey()] = r
 			}
+		}
 		errs <- nil
 	}()
 
@@ -146,16 +150,6 @@ func (hm *HashMap) Injected() bool {
 	return false
 }
 
-// Maintain runs a light maintenance operation on the database.
-func (hm *HashMap) Maintain(_ context.Context) error {
-	return nil
-}
-
-// MaintainThorough runs a thorough maintenance operation on the database.
-func (hm *HashMap) MaintainThorough(_ context.Context) error {
-	return nil
-}
-
 // MaintainRecordStates maintains records states in the database.
 func (hm *HashMap) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	hm.dbLock.Lock()

database/storage/hashmap/map_test.go

@@ -2,15 +2,22 @@
 package hashmap
 
 import (
-	"context"
 	"reflect"
 	"sync"
 	"testing"
 
+	"github.com/safing/portbase/database/storage"
+
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
 )
 
+var (
+	// Compile time interface checks.
+	_ storage.Interface = &HashMap{}
+	_ storage.Batcher   = &HashMap{}
+)
+
 type TestRecord struct {
 	record.Base
 	sync.Mutex
@@ -130,16 +137,6 @@ func TestHashMap(t *testing.T) {
 		t.Fatal("should fail")
 	}
 
-	// maintenance
-	err = db.Maintain(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db.MaintainThorough(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// shutdown
 	err = db.Shutdown()
 	if err != nil {

database/storage/injectbase.go

@@ -28,7 +28,7 @@ func (i *InjectBase) Put(m record.Record) (record.Record, error) {
 }
 
 // PutMany stores many records in the database.
-func (i *InjectBase) PutMany() (batch chan record.Record, err chan error) {
+func (i *InjectBase) PutMany(shadowDelete bool) (batch chan record.Record, err chan error) {
 	batch = make(chan record.Record)
 	err = make(chan error, 1)
 	err <- errNotImplemented

database/storage/interface.go

@@ -11,20 +11,28 @@ import (
 
 // Interface defines the database storage API.
 type Interface interface {
+	// Primary Interface
 	Get(key string) (record.Record, error)
 	Put(m record.Record) (record.Record, error)
 	Delete(key string) error
 	Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
 
+	// Information and Control
 	ReadOnly() bool
 	Injected() bool
+	Shutdown() error
+
+	// Mandatory Record Maintenance
+	MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error
+}
+
+// Maintainer defines the database storage API for backends that require regular maintenance.
+type Maintainer interface {
 	Maintain(ctx context.Context) error
 	MaintainThorough(ctx context.Context) error
-	MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error
-	Shutdown() error
 }
 
 // Batcher defines the database storage API for backends that support batch operations.
 type Batcher interface {
-	PutMany() (batch chan<- record.Record, errs <-chan error)
+	PutMany(shadowDelete bool) (batch chan<- record.Record, errs <-chan error)
 }

notifications/database.go

@@ -48,7 +48,6 @@ func registerAsDatabase() error {
 		Name:        "notifications",
 		Description: "Notifications",
 		StorageType: "injected",
-		PrimaryAPI:  "",
 	})
 	if err != nil {
 		return err