mirror of
https://github.com/safing/portbase
synced 2025-09-09 13:55:47 +00:00
Move DB record maintenance to storage interface
This commit is contained in:
Daniel
parent
4eb21405cc
commit
bea130d755
16 changed files with 385 additions and 224 deletions
|
|
@ -11,9 +11,11 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/safing/portbase/database/record"
|
||||
"github.com/safing/portbase/database/storage"
|
||||
|
||||
q "github.com/safing/portbase/database/query"
|
||||
"github.com/safing/portbase/database/record"
|
||||
|
||||
_ "github.com/safing/portbase/database/storage/badger"
|
||||
_ "github.com/safing/portbase/database/storage/bbolt"
|
||||
_ "github.com/safing/portbase/database/storage/fstree"
|
||||
|
|
@ -24,117 +26,195 @@ func makeKey(dbName, key string) string {
|
|||
return fmt.Sprintf("%s:%s", dbName, key)
|
||||
}
|
||||
|
||||
func testDatabase(t *testing.T, storageType string) {
|
||||
dbName := fmt.Sprintf("testing-%s", storageType)
|
||||
_, err := Register(&Database{
|
||||
Name: dbName,
|
||||
Description: fmt.Sprintf("Unit Test Database for %s", storageType),
|
||||
StorageType: storageType,
|
||||
PrimaryAPI: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
func testDatabase(t *testing.T, storageType string, testPutMany, testRecordMaintenance bool) { //nolint:gocognit,gocyclo
|
||||
t.Run(fmt.Sprintf("TestStorage_%s", storageType), func(t *testing.T) {
|
||||
dbName := fmt.Sprintf("testing-%s", storageType)
|
||||
fmt.Println(dbName)
|
||||
_, err := Register(&Database{
|
||||
Name: dbName,
|
||||
Description: fmt.Sprintf("Unit Test Database for %s", storageType),
|
||||
StorageType: storageType,
|
||||
PrimaryAPI: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dbController, err := getController(dbName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// hook
|
||||
hook, err := RegisterHook(q.New(dbName).MustBeValid(), &HookBase{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// hook
|
||||
hook, err := RegisterHook(q.New(dbName).MustBeValid(), &HookBase{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// interface
|
||||
db := NewInterface(&Options{
|
||||
Local: true,
|
||||
Internal: true,
|
||||
})
|
||||
// interface
|
||||
db := NewInterface(&Options{
|
||||
Local: true,
|
||||
Internal: true,
|
||||
})
|
||||
|
||||
// sub
|
||||
sub, err := db.Subscribe(q.New(dbName).MustBeValid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// sub
|
||||
sub, err := db.Subscribe(q.New(dbName).MustBeValid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
A := NewExample(makeKey(dbName, "A"), "Herbert", 411)
|
||||
err = A.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
A := NewExample(dbName+":A", "Herbert", 411)
|
||||
err = A.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
B := NewExample(makeKey(dbName, "B"), "Fritz", 347)
|
||||
err = B.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
B := NewExample(makeKey(dbName, "B"), "Fritz", 347)
|
||||
err = B.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
C := NewExample(makeKey(dbName, "C"), "Norbert", 217)
|
||||
err = C.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
C := NewExample(makeKey(dbName, "C"), "Norbert", 217)
|
||||
err = C.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
exists, err := db.Exists(makeKey(dbName, "A"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("record %s should exist!", makeKey(dbName, "A"))
|
||||
}
|
||||
exists, err := db.Exists(makeKey(dbName, "A"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("record %s should exist!", makeKey(dbName, "A"))
|
||||
}
|
||||
|
||||
A1, err := GetExample(makeKey(dbName, "A"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(A, A1) {
|
||||
log.Fatalf("A and A1 mismatch, A1: %v", A1)
|
||||
}
|
||||
A1, err := GetExample(makeKey(dbName, "A"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(A, A1) {
|
||||
log.Fatalf("A and A1 mismatch, A1: %v", A1)
|
||||
}
|
||||
|
||||
query, err := q.New(dbName).Where(
|
||||
q.And(
|
||||
q.Where("Name", q.EndsWith, "bert"),
|
||||
q.Where("Score", q.GreaterThan, 100),
|
||||
),
|
||||
).Check()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cnt := countRecords(t, db, q.New(dbName).Where(
|
||||
q.And(
|
||||
q.Where("Name", q.EndsWith, "bert"),
|
||||
q.Where("Score", q.GreaterThan, 100),
|
||||
),
|
||||
))
|
||||
if cnt != 2 {
|
||||
t.Fatalf("expected two records, got %d", cnt)
|
||||
}
|
||||
|
||||
it, err := db.Query(query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// test putmany
|
||||
if testPutMany {
|
||||
batchPut := db.PutMany(dbName)
|
||||
records := []record.Record{A, B, C, nil} // nil is to signify finish
|
||||
for _, r := range records {
|
||||
err = batchPut(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cnt := 0
|
||||
for range it.Next {
|
||||
cnt++
|
||||
}
|
||||
if it.Err() != nil {
|
||||
t.Fatal(it.Err())
|
||||
}
|
||||
if cnt != 2 {
|
||||
t.Fatalf("expected two records, got %d", cnt)
|
||||
}
|
||||
// test maintenance
|
||||
if testRecordMaintenance {
|
||||
now := time.Now().UTC()
|
||||
nowUnix := now.Unix()
|
||||
|
||||
switch storageType {
|
||||
case "bbolt", "hashmap":
|
||||
batchPut := db.PutMany(dbName)
|
||||
records := []record.Record{A, B, C, nil} // nil is to signify finish
|
||||
for _, r := range records {
|
||||
err = batchPut(r)
|
||||
// we start with 3 records without expiry
|
||||
cnt := countRecords(t, db, q.New(dbName))
|
||||
if cnt != 3 {
|
||||
t.Fatalf("expected three records, got %d", cnt)
|
||||
}
|
||||
// delete entry
|
||||
A.Meta().Deleted = nowUnix - 61
|
||||
err = A.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// expire entry
|
||||
B.Meta().Expires = nowUnix - 1
|
||||
err = B.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// one left
|
||||
cnt = countRecords(t, db, q.New(dbName))
|
||||
if cnt != 1 {
|
||||
t.Fatalf("expected one record, got %d", cnt)
|
||||
}
|
||||
|
||||
// run maintenance
|
||||
err = dbController.MaintainRecordStates(context.TODO(), now.Add(-60*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// one left
|
||||
cnt = countRecords(t, db, q.New(dbName))
|
||||
if cnt != 1 {
|
||||
t.Fatalf("expected one record, got %d", cnt)
|
||||
}
|
||||
|
||||
// check status individually
|
||||
_, err = dbController.storage.Get("A")
|
||||
if err != storage.ErrNotFound {
|
||||
t.Errorf("A should be deleted and purged, err=%s", err)
|
||||
}
|
||||
B1, err := dbController.storage.Get("B")
|
||||
if err != nil {
|
||||
t.Fatalf("should exist: %s, original meta: %+v", err, B.Meta())
|
||||
}
|
||||
if B1.Meta().Deleted == 0 {
|
||||
t.Errorf("B should be deleted")
|
||||
}
|
||||
|
||||
// delete last entry
|
||||
C.Meta().Deleted = nowUnix - 1
|
||||
err = C.Save()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// run maintenance
|
||||
err = dbController.MaintainRecordStates(context.TODO(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// check status individually
|
||||
B2, err := dbController.storage.Get("B")
|
||||
if err == nil {
|
||||
t.Errorf("B should be deleted and purged, meta: %+v", B2.Meta())
|
||||
} else if err != storage.ErrNotFound {
|
||||
t.Errorf("B should be deleted and purged, err=%s", err)
|
||||
}
|
||||
C2, err := dbController.storage.Get("C")
|
||||
if err == nil {
|
||||
t.Errorf("C should be deleted and purged, meta: %+v", C2.Meta())
|
||||
} else if err != storage.ErrNotFound {
|
||||
t.Errorf("C should be deleted and purged, err=%s", err)
|
||||
}
|
||||
|
||||
// none left
|
||||
cnt = countRecords(t, db, q.New(dbName))
|
||||
if cnt != 0 {
|
||||
t.Fatalf("expected no records, got %d", cnt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = hook.Cancel()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = sub.Cancel()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = hook.Cancel()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = sub.Cancel()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func TestDatabaseSystem(t *testing.T) {
|
||||
|
|
@ -158,22 +238,22 @@ func TestDatabaseSystem(t *testing.T) {
|
|||
}
|
||||
defer os.RemoveAll(testDir) // clean up
|
||||
|
||||
testDatabase(t, "badger")
|
||||
testDatabase(t, "bbolt")
|
||||
testDatabase(t, "fstree")
|
||||
testDatabase(t, "hashmap")
|
||||
testDatabase(t, "bbolt", true, true)
|
||||
testDatabase(t, "hashmap", true, true)
|
||||
testDatabase(t, "fstree", false, false)
|
||||
testDatabase(t, "badger", false, false)
|
||||
|
||||
err = MaintainRecordStates(context.TODO())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = Maintain()
|
||||
err = Maintain(context.TODO())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = MaintainThorough()
|
||||
err = MaintainThorough(context.TODO())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -182,5 +262,25 @@ func TestDatabaseSystem(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func countRecords(t *testing.T, db *Interface, query *q.Query) int {
|
||||
_, err := query.Check()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
it, err := db.Query(query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cnt := 0
|
||||
for range it.Next {
|
||||
cnt++
|
||||
}
|
||||
if it.Err() != nil {
|
||||
t.Fatal(it.Err())
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue