Merge pull request #82 from safing/feature/deletemany

Make shadow deletes conditional

config/database.go

@@ -134,7 +134,6 @@ func registerAsDatabase() error {
 		Name:        "config",
 		Description: "Configuration Manager",
 		StorageType: "injected",
-		PrimaryAPI:  "",
 	})
 	if err != nil {
 		return err

database/controller.go

@@ -16,7 +16,8 @@ import (
 
 // A Controller takes care of all the extra database logic.
 type Controller struct {
-	storage storage.Interface
+	storage      storage.Interface
+	shadowDelete bool
 
 	hooks         []*RegisteredHook
 	subscriptions []*Subscription
@@ -33,11 +34,12 @@ type Controller struct {
 }
 
 // newController creates a new controller for a storage.
-func newController(storageInt storage.Interface) *Controller {
+func newController(storageInt storage.Interface, shadowDelete bool) *Controller {
 	return &Controller{
-		storage:     storageInt,
-		migrating:   abool.NewBool(false),
-		hibernating: abool.NewBool(false),
+		storage:      storageInt,
+		shadowDelete: shadowDelete,
+		migrating:    abool.NewBool(false),
+		hibernating:  abool.NewBool(false),
 	}
 }
 
@@ -122,12 +124,21 @@ func (c *Controller) Put(r record.Record) (err error) {
 		}
 	}
 
-	r, err = c.storage.Put(r)
-	if err != nil {
-		return err
-	}
-	if r == nil {
-		return errors.New("storage returned nil record after successful put operation")
+	if !c.shadowDelete && r.Meta().IsDeleted() {
+		// Immediate delete.
+		err = c.storage.Delete(r.DatabaseKey())
+		if err != nil {
+			return err
+		}
+	} else {
+		// Put or shadow delete.
+		r, err = c.storage.Put(r)
+		if err != nil {
+			return err
+		}
+		if r == nil {
+			return errors.New("storage returned nil record after successful put operation")
+		}
 	}
 
 	// process subscriptions
@@ -161,7 +172,7 @@ func (c *Controller) PutMany() (chan<- record.Record, <-chan error) {
 	}
 
 	if batcher, ok := c.storage.(storage.Batcher); ok {
-		return batcher.PutMany()
+		return batcher.PutMany(c.shadowDelete)
 	}
 
 	errs := make(chan error, 1)
@@ -236,7 +247,10 @@ func (c *Controller) Maintain(ctx context.Context) error {
 		return nil
 	}
 
-	return c.storage.Maintain(ctx)
+	if maintainer, ok := c.storage.(storage.Maintainer); ok {
+		return maintainer.Maintain(ctx)
+	}
+	return nil
 }
 
 // MaintainThorough runs the MaintainThorough method on the storage.
@@ -248,7 +262,10 @@ func (c *Controller) MaintainThorough(ctx context.Context) error {
 		return nil
 	}
 
-	return c.storage.MaintainThorough(ctx)
+	if maintainer, ok := c.storage.(storage.Maintainer); ok {
+		return maintainer.MaintainThorough(ctx)
+	}
+	return nil
 }
 
 // MaintainRecordStates runs the record state lifecycle maintenance on the storage.

database/controllers.go

@@ -51,7 +51,7 @@ func getController(name string) (*Controller, error) {
 		return nil, fmt.Errorf(`could not start database %s (type %s): %s`, name, registeredDB.StorageType, err)
 	}
 
-	controller = newController(storageInt)
+	controller = newController(storageInt, registeredDB.ShadowDelete)
 	controllers[name] = controller
 	return controller, nil
 }
@@ -82,7 +82,7 @@ func InjectDatabase(name string, storageInt storage.Interface) (*Controller, err
 		return nil, fmt.Errorf(`database not of type "injected"`)
 	}
 
-	controller := newController(storageInt)
+	controller := newController(storageInt, false)
 	controllers[name] = controller
 	return controller, nil
 }

database/database.go

@@ -7,13 +7,13 @@ import (
 
 // Database holds information about registered databases
 type Database struct {
-	Name        string
-	Description string
-	StorageType string
-	PrimaryAPI  string
-	Registered  time.Time
-	LastUpdated time.Time
-	LastLoaded  time.Time
+	Name         string
+	Description  string
+	StorageType  string
+	ShadowDelete bool // Whether deleted records should be kept until purged.
+	Registered   time.Time
+	LastUpdated  time.Time
+	LastLoaded   time.Time
 }
 
 // MigrateTo migrates the database to another storage type.

database/database_test.go

@@ -31,10 +31,10 @@ func testDatabase(t *testing.T, storageType string, testPutMany, testRecordMaint
 		dbName := fmt.Sprintf("testing-%s", storageType)
 		fmt.Println(dbName)
 		_, err := Register(&Database{
-			Name:        dbName,
-			Description: fmt.Sprintf("Unit Test Database for %s", storageType),
-			StorageType: storageType,
-			PrimaryAPI:  "",
+			Name:         dbName,
+			Description:  fmt.Sprintf("Unit Test Database for %s", storageType),
+			StorageType:  storageType,
+			ShadowDelete: true,
 		})
 		if err != nil {
 			t.Fatal(err)

database/registry.go

@@ -49,8 +49,8 @@ func Register(new *Database) (*Database, error) {
 			registeredDB.Description = new.Description
 			save = true
 		}
-		if registeredDB.PrimaryAPI != new.PrimaryAPI {
-			registeredDB.PrimaryAPI = new.PrimaryAPI
+		if registeredDB.ShadowDelete != new.ShadowDelete {
+			registeredDB.ShadowDelete = new.ShadowDelete
 			save = true
 		}
 	} else {

database/storage/badger/badger_test.go

@@ -11,6 +11,13 @@ import (
 
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/database/storage"
+)
+
+var (
+	// Compile time interface checks.
+	_ storage.Interface  = &Badger{}
+	_ storage.Maintainer = &Badger{}
 )
 
 type TestRecord struct {
@@ -117,13 +124,18 @@ func TestBadger(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.Maintain(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db.MaintainThorough(context.TODO())
-	if err != nil {
-		t.Fatal(err)
+	maintainer, ok := db.(storage.Maintainer)
+	if ok {
+		err = maintainer.Maintain(context.TODO())
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = maintainer.MaintainThorough(context.TODO())
+		if err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		t.Fatal("should implement Maintainer")
 	}
 
 	// shutdown

database/storage/bbolt/bbolt.go

@@ -107,7 +107,7 @@ func (b *BBolt) Put(r record.Record) (record.Record, error) {
 }
 
 // PutMany stores many records in the database.
-func (b *BBolt) PutMany() (chan<- record.Record, <-chan error) {
+func (b *BBolt) PutMany(shadowDelete bool) (chan<- record.Record, <-chan error) {
 	batch := make(chan record.Record, 100)
 	errs := make(chan error, 1)
 
@@ -115,16 +115,26 @@ func (b *BBolt) PutMany() (chan<- record.Record, <-chan error) {
 		err := b.db.Batch(func(tx *bbolt.Tx) error {
 			bucket := tx.Bucket(bucketName)
 			for r := range batch {
-				// marshal
-				data, txErr := r.MarshalRecord(r)
-				if txErr != nil {
-					return txErr
-				}
+				if !shadowDelete && r.Meta().IsDeleted() {
+					// Immediate delete.
+					txErr := bucket.Delete([]byte(r.DatabaseKey()))
+					if txErr != nil {
+						return txErr
+					}
+				} else {
+					// Put or shadow delete.
 
-				// put
-				txErr = bucket.Put([]byte(r.DatabaseKey()), data)
-				if txErr != nil {
-					return txErr
+					// marshal
+					data, txErr := r.MarshalRecord(r)
+					if txErr != nil {
+						return txErr
+					}
+
+					// put
+					txErr = bucket.Put([]byte(r.DatabaseKey()), data)
+					if txErr != nil {
+						return txErr
+					}
 				}
 			}
 			return nil
@@ -235,16 +245,6 @@ func (b *BBolt) Injected() bool {
 	return false
 }
 
-// Maintain runs a light maintenance operation on the database.
-func (b *BBolt) Maintain(_ context.Context) error {
-	return nil
-}
-
-// MaintainThorough runs a thorough maintenance operation on the database.
-func (b *BBolt) MaintainThorough(_ context.Context) error {
-	return nil
-}
-
 // MaintainRecordStates maintains records states in the database.
 func (b *BBolt) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	now := time.Now().Unix()

database/storage/bbolt/bbolt_test.go

@@ -12,6 +12,13 @@ import (
 
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/database/storage"
+)
+
+var (
+	// Compile time interface checks.
+	_ storage.Interface = &BBolt{}
+	_ storage.Batcher   = &BBolt{}
 )
 
 type TestRecord struct {
@@ -146,14 +153,6 @@ func TestBBolt(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.Maintain(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db.MaintainThorough(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
 	err = db.MaintainRecordStates(context.TODO(), time.Now())
 	if err != nil {
 		t.Fatal(err)

database/storage/fstree/fstree.go

@@ -255,16 +255,6 @@ func (fst *FSTree) Injected() bool {
 	return false
 }
 
-// Maintain runs a light maintenance operation on the database.
-func (fst *FSTree) Maintain(_ context.Context) error {
-	return nil
-}
-
-// MaintainThorough runs a thorough maintenance operation on the database.
-func (fst *FSTree) MaintainThorough(_ context.Context) error {
-	return nil
-}
-
 // MaintainRecordStates maintains records states in the database.
 func (fst *FSTree) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	// TODO: implement MaintainRecordStates
@@ -279,6 +269,7 @@ func (fst *FSTree) Shutdown() error {
 // writeFile mirrors ioutil.WriteFile, replacing an existing file with the same
 // name atomically. This is not atomic on Windows, but still an improvement.
 // TODO: Replace with github.com/google/renamio.WriteFile as soon as it is fixed on Windows.
+// TODO: This has become a wont-fix. Explore other options.
 // This function is forked from https://github.com/google/renameio/blob/a368f9987532a68a3d676566141654a81aa8100b/writefile.go.
 func writeFile(filename string, data []byte, perm os.FileMode) error {
 	t, err := renameio.TempFile("", filename)

database/storage/fstree/fstree_test.go

@@ -0,0 +1,8 @@
+package fstree
+
+import "github.com/safing/portbase/database/storage"
+
+var (
+	// Compile time interface checks.
+	_ storage.Interface = &FSTree{}
+)

database/storage/hashmap/map.go

@@ -54,7 +54,7 @@ func (hm *HashMap) Put(r record.Record) (record.Record, error) {
 }
 
 // PutMany stores many records in the database.
-func (hm *HashMap) PutMany() (chan<- record.Record, <-chan error) {
+func (hm *HashMap) PutMany(shadowDelete bool) (chan<- record.Record, <-chan error) {
 	hm.dbLock.Lock()
 	defer hm.dbLock.Unlock()
 	// we could lock for every record, but we want to have the same behaviour
@@ -66,7 +66,11 @@ func (hm *HashMap) PutMany() (chan<- record.Record, <-chan error) {
 	// start handler
 	go func() {
 		for r := range batch {
-			hm.db[r.DatabaseKey()] = r
+			if !shadowDelete && r.Meta().IsDeleted() {
+				delete(hm.db, r.DatabaseKey())
+			} else {
+				hm.db[r.DatabaseKey()] = r
+			}
 		}
 		errs <- nil
 	}()
@@ -146,16 +150,6 @@ func (hm *HashMap) Injected() bool {
 	return false
 }
 
-// Maintain runs a light maintenance operation on the database.
-func (hm *HashMap) Maintain(_ context.Context) error {
-	return nil
-}
-
-// MaintainThorough runs a thorough maintenance operation on the database.
-func (hm *HashMap) MaintainThorough(_ context.Context) error {
-	return nil
-}
-
 // MaintainRecordStates maintains records states in the database.
 func (hm *HashMap) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error {
 	hm.dbLock.Lock()

database/storage/hashmap/map_test.go

@@ -2,15 +2,22 @@
 package hashmap
 
 import (
-	"context"
 	"reflect"
 	"sync"
 	"testing"
 
+	"github.com/safing/portbase/database/storage"
+
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
 )
 
+var (
+	// Compile time interface checks.
+	_ storage.Interface = &HashMap{}
+	_ storage.Batcher   = &HashMap{}
+)
+
 type TestRecord struct {
 	record.Base
 	sync.Mutex
@@ -130,16 +137,6 @@ func TestHashMap(t *testing.T) {
 		t.Fatal("should fail")
 	}
 
-	// maintenance
-	err = db.Maintain(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db.MaintainThorough(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// shutdown
 	err = db.Shutdown()
 	if err != nil {

database/storage/injectbase.go

@@ -28,7 +28,7 @@ func (i *InjectBase) Put(m record.Record) (record.Record, error) {
 }
 
 // PutMany stores many records in the database.
-func (i *InjectBase) PutMany() (batch chan record.Record, err chan error) {
+func (i *InjectBase) PutMany(shadowDelete bool) (batch chan record.Record, err chan error) {
 	batch = make(chan record.Record)
 	err = make(chan error, 1)
 	err <- errNotImplemented

database/storage/interface.go

@@ -11,20 +11,28 @@ import (
 
 // Interface defines the database storage API.
 type Interface interface {
+	// Primary Interface
 	Get(key string) (record.Record, error)
 	Put(m record.Record) (record.Record, error)
 	Delete(key string) error
 	Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
 
+	// Information and Control
 	ReadOnly() bool
 	Injected() bool
+	Shutdown() error
+
+	// Mandatory Record Maintenance
+	MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error
+}
+
+// Maintainer defines the database storage API for backends that require regular maintenance.
+type Maintainer interface {
 	Maintain(ctx context.Context) error
 	MaintainThorough(ctx context.Context) error
-	MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time) error
-	Shutdown() error
 }
 
 // Batcher defines the database storage API for backends that support batch operations.
 type Batcher interface {
-	PutMany() (batch chan<- record.Record, errs <-chan error)
+	PutMany(shadowDelete bool) (batch chan<- record.Record, errs <-chan error)
 }

notifications/database.go

@@ -48,7 +48,6 @@ func registerAsDatabase() error {
 		Name:        "notifications",
 		Description: "Notifications",
 		StorageType: "injected",
-		PrimaryAPI:  "",
 	})
 	if err != nil {
 		return err