From ed61819be751e69a2da1aed2d5ebb24a71672507 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 25 Oct 2019 13:37:42 +0200 Subject: [PATCH] Cleanup database package --- database/registry.go | 1 + database/storage/bbolt/bbolt_test.go | 2 +- database/storage/hashmap/map.go | 140 +++++++++++++++++++++++++ database/storage/hashmap/map_test.go | 147 +++++++++++++++++++++++++++ 4 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 database/storage/hashmap/map.go create mode 100644 database/storage/hashmap/map_test.go diff --git a/database/registry.go b/database/registry.go index 23d382c..b6d1984 100644 --- a/database/registry.go +++ b/database/registry.go @@ -139,6 +139,7 @@ func saveRegistry(lock bool) error { } // write file + // FIXME: write atomically (best effort) filePath := path.Join(rootStructure.Path, registryFileName) return ioutil.WriteFile(filePath, data, 0600) } diff --git a/database/storage/bbolt/bbolt_test.go b/database/storage/bbolt/bbolt_test.go index 5264f56..e5cdd3e 100644 --- a/database/storage/bbolt/bbolt_test.go +++ b/database/storage/bbolt/bbolt_test.go @@ -31,7 +31,7 @@ type TestRecord struct { B bool } -func TestBadger(t *testing.T) { +func TestBBolt(t *testing.T) { testDir, err := ioutil.TempDir("", "testing-") if err != nil { t.Fatal(err) diff --git a/database/storage/hashmap/map.go b/database/storage/hashmap/map.go new file mode 100644 index 0000000..c6bb65d --- /dev/null +++ b/database/storage/hashmap/map.go @@ -0,0 +1,140 @@ +package hashmap + +import ( + "errors" + "fmt" + "sync" + "time" + + "github.com/safing/portbase/database/iterator" + "github.com/safing/portbase/database/query" + "github.com/safing/portbase/database/record" + "github.com/safing/portbase/database/storage" +) + +// HashMap storage. +type HashMap struct { + name string + db map[string]record.Record + dbLock sync.RWMutex +} + +func init() { + _ = storage.Register("hashmap", NewHashMap) +} + +// NewHashMap creates a hashmap database. +func NewHashMap(name, location string) (storage.Interface, error) { + return &HashMap{ + name: name, + db: make(map[string]record.Record), + }, nil +} + +// Get returns a database record. +func (hm *HashMap) Get(key string) (record.Record, error) { + hm.dbLock.RLock() + defer hm.dbLock.RUnlock() + + r, ok := hm.db[key] + if !ok { + return nil, storage.ErrNotFound + } + return r, nil +} + +// Put stores a record in the database. +func (hm *HashMap) Put(r record.Record) error { + hm.dbLock.Lock() + defer hm.dbLock.Unlock() + + hm.db[r.DatabaseKey()] = r + return nil +} + +// Delete deletes a record from the database. +func (hm *HashMap) Delete(key string) error { + hm.dbLock.Lock() + defer hm.dbLock.Unlock() + + delete(hm.db, key) + return nil +} + +// Query returns a an iterator for the supplied query. +func (hm *HashMap) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) { + _, err := q.Check() + if err != nil { + return nil, fmt.Errorf("invalid query: %s", err) + } + + queryIter := iterator.New() + + go hm.queryExecutor(queryIter, q, local, internal) + return queryIter, nil +} + +func (hm *HashMap) queryExecutor(queryIter *iterator.Iterator, q *query.Query, local, internal bool) { + hm.dbLock.RLock() + defer hm.dbLock.RUnlock() + + var err error + +mapLoop: + for key, record := range hm.db { + + switch { + case !q.MatchesKey(key): + continue + case !q.MatchesRecord(record): + continue + case !record.Meta().CheckValidity(): + continue + case !record.Meta().CheckPermission(local, internal): + continue + } + + select { + case <-queryIter.Done: + break mapLoop + case queryIter.Next <- record: + default: + select { + case <-queryIter.Done: + break mapLoop + case queryIter.Next <- record: + case <-time.After(1 * time.Second): + err = errors.New("query timeout") + break mapLoop + } + } + + } + + queryIter.Finish(err) +} + +// ReadOnly returns whether the database is read only. +func (hm *HashMap) ReadOnly() bool { + return false +} + +// Injected returns whether the database is injected. +func (hm *HashMap) Injected() bool { + return false +} + +// Maintain runs a light maintenance operation on the database. +func (hm *HashMap) Maintain() error { + return nil +} + +// MaintainThorough runs a thorough maintenance operation on the database. +func (hm *HashMap) MaintainThorough() (err error) { + return nil +} + +// Shutdown shuts down the database. +func (hm *HashMap) Shutdown() error { + return nil +} diff --git a/database/storage/hashmap/map_test.go b/database/storage/hashmap/map_test.go new file mode 100644 index 0000000..911dc5b --- /dev/null +++ b/database/storage/hashmap/map_test.go @@ -0,0 +1,147 @@ +//nolint:unparam,maligned +package hashmap + +import ( + "reflect" + "sync" + "testing" + + "github.com/safing/portbase/database/query" + "github.com/safing/portbase/database/record" +) + +type TestRecord struct { + record.Base + sync.Mutex + S string + I int + I8 int8 + I16 int16 + I32 int32 + I64 int64 + UI uint + UI8 uint8 + UI16 uint16 + UI32 uint32 + UI64 uint64 + F32 float32 + F64 float64 + B bool +} + +func TestHashMap(t *testing.T) { + // start + db, err := NewHashMap("test", "") + if err != nil { + t.Fatal(err) + } + + a := &TestRecord{ + S: "banana", + I: 42, + I8: 42, + I16: 42, + I32: 42, + I64: 42, + UI: 42, + UI8: 42, + UI16: 42, + UI32: 42, + UI64: 42, + F32: 42.42, + F64: 42.42, + B: true, + } + a.SetMeta(&record.Meta{}) + a.Meta().Update() + a.SetKey("test:A") + + // put record + err = db.Put(a) + if err != nil { + t.Fatal(err) + } + + // get and compare + a1, err := db.Get("A") + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(a, a1) { + t.Fatalf("mismatch, got %v", a1) + } + + // setup query test records + qA := &TestRecord{} + qA.SetKey("test:path/to/A") + qA.CreateMeta() + qB := &TestRecord{} + qB.SetKey("test:path/to/B") + qB.CreateMeta() + qC := &TestRecord{} + qC.SetKey("test:path/to/C") + qC.CreateMeta() + qZ := &TestRecord{} + qZ.SetKey("test:z") + qZ.CreateMeta() + // put + err = db.Put(qA) + if err == nil { + err = db.Put(qB) + } + if err == nil { + err = db.Put(qC) + } + if err == nil { + err = db.Put(qZ) + } + if err != nil { + t.Fatal(err) + } + + // test query + q := query.New("test:path/to/").MustBeValid() + it, err := db.Query(q, true, true) + if err != nil { + t.Fatal(err) + } + cnt := 0 + for range it.Next { + cnt++ + } + if it.Err() != nil { + t.Fatal(it.Err()) + } + if cnt != 3 { + t.Fatalf("unexpected query result count: %d", cnt) + } + + // delete + err = db.Delete("A") + if err != nil { + t.Fatal(err) + } + + // check if its gone + _, err = db.Get("A") + if err == nil { + t.Fatal("should fail") + } + + // maintenance + err = db.Maintain() + if err != nil { + t.Fatal(err) + } + err = db.MaintainThorough() + if err != nil { + t.Fatal(err) + } + + // shutdown + err = db.Shutdown() + if err != nil { + t.Fatal(err) + } +}