From efabb291d794ccc887c6e1139d36be7e47f35fdd Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 4 Sep 2018 17:09:32 +0200 Subject: [PATCH] Work on database revamp [WIP] --- database/base.go | 63 ---- database/database.go | 57 ++-- database/databases.go | 56 ++++ database/dbutils/wrapper.go | 67 ---- database/interface.go | 17 + database/model.go | 2 - database/model/base.go | 47 +++ database/model/formats.go | 15 + database/model/key.go | 14 + database/model/meta-bench_test.go | 466 ++++++++++++++++++++++++++++ database/model/meta-gencode.go | 157 ++++++++++ database/model/meta-gencode_test.go | 35 +++ database/model/meta.gencode.gen.go | 163 ---------- database/model/meta.go | 74 ++++- database/model/meta_test.go | 311 ------------------- database/model/model.go | 7 + database/model/wrapper.go | 67 ++++ database/storage/interface.go | 42 +-- database/storages.go | 36 +++ 19 files changed, 1032 insertions(+), 664 deletions(-) delete mode 100644 database/base.go create mode 100644 database/databases.go delete mode 100644 database/dbutils/wrapper.go create mode 100644 database/interface.go create mode 100644 database/model/base.go create mode 100644 database/model/formats.go create mode 100644 database/model/key.go create mode 100644 database/model/meta-bench_test.go create mode 100644 database/model/meta-gencode.go create mode 100644 database/model/meta-gencode_test.go delete mode 100644 database/model/meta.gencode.gen.go delete mode 100644 database/model/meta_test.go create mode 100644 database/model/wrapper.go create mode 100644 database/storages.go diff --git a/database/base.go b/database/base.go deleted file mode 100644 index a9bf424..0000000 --- a/database/base.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file. - -package database - -import ( - "errors" - "strings" - - "github.com/Safing/safing-core/database/dbutils" - - "github.com/ipfs/go-datastore" - uuid "github.com/satori/go.uuid" -) - -type Base struct { - dbKey *datastore.Key - meta *dbutils.Meta -} - -func (m *Base) SetKey(key *datastore.Key) { - m.dbKey = key -} - -func (m *Base) GetKey() *datastore.Key { - return m.dbKey -} - -func (m *Base) FmtKey() string { - return m.dbKey.String() -} - -func (m *Base) Meta() *dbutils.Meta { - return m.meta -} - -func (m *Base) CreateObject(namespace *datastore.Key, name string, model Model) error { - var newKey datastore.Key - if name == "" { - newKey = NewInstance(namespace.ChildString(getTypeName(model)), strings.Replace(uuid.NewV4().String(), "-", "", -1)) - } else { - newKey = NewInstance(namespace.ChildString(getTypeName(model)), name) - } - m.dbKey = &newKey - return Create(*m.dbKey, model) -} - -func (m *Base) SaveObject(model Model) error { - if m.dbKey == nil { - return errors.New("cannot save new object, use Create() instead") - } - return Update(*m.dbKey, model) -} - -func (m *Base) Delete() error { - if m.dbKey == nil { - return errors.New("cannot delete object unsaved object") - } - return Delete(*m.dbKey) -} - -func NewInstance(k datastore.Key, s string) datastore.Key { - return datastore.NewKey(k.String() + ":" + s) -} diff --git a/database/database.go b/database/database.go index b1b7a5f..863604e 100644 --- a/database/database.go +++ b/database/database.go @@ -9,29 +9,20 @@ import ( "path" "strings" - ds "github.com/ipfs/go-datastore" - dsq "github.com/ipfs/go-datastore/query" - mount "github.com/ipfs/go-datastore/syncmount" - "github.com/Safing/safing-core/database/dbutils" - "github.com/Safing/safing-core/database/ds/channelshim" - "github.com/Safing/safing-core/database/ds/leveldb" "github.com/Safing/safing-core/log" "github.com/Safing/safing-core/meta" ) -// TODO: do not let other modules panic, even if database module crashes. -var db ds.Datastore - var ErrNotFound = errors.New("database: entry could not be found") func init() { - if strings.HasSuffix(os.Args[0], ".test") { - // testing setup - log.Warning("===== DATABASE RUNNING IN TEST MODE =====") - db = channelshim.NewChanneledDatastore(ds.NewMapDatastore()) - return - } + // if strings.HasSuffix(os.Args[0], ".test") { + // // testing setup + // log.Warning("===== DATABASE RUNNING IN TEST MODE =====") + // db = channelshim.NewChanneledDatastore(ds.NewMapDatastore()) + // return + // } // sfsDB, err := simplefs.NewDatastore(meta.DatabaseDir()) // if err != nil { @@ -39,24 +30,24 @@ func init() { // os.Exit(1) // } - ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{}) - if err != nil { - fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err) - os.Exit(1) - } - - mapDB := ds.NewMapDatastore() - - db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{ - mount.Mount{ - Prefix: ds.NewKey("/Run"), - Datastore: mapDB, - }, - mount.Mount{ - Prefix: ds.NewKey("/"), - Datastore: ldb, - }, - })) + // ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{}) + // if err != nil { + // fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err) + // os.Exit(1) + // } + // + // mapDB := ds.NewMapDatastore() + // + // db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{ + // mount.Mount{ + // Prefix: ds.NewKey("/Run"), + // Datastore: mapDB, + // }, + // mount.Mount{ + // Prefix: ds.NewKey("/"), + // Datastore: ldb, + // }, + // })) } diff --git a/database/databases.go b/database/databases.go new file mode 100644 index 0000000..d4643f5 --- /dev/null +++ b/database/databases.go @@ -0,0 +1,56 @@ +package database + + +var ( + databases = make(map[string]*storage.Interface) + databasesLock sync.Mutex +) + +func getDatabase(name string) *storage.Interface { + databasesLock.Lock() + defer databasesLock.Unlock() + storage, ok := databases[name] + if ok { + return + } +} + +func databaseExists(name string) (exists bool) { + // check if folder exists + return true +} + +// CreateDatabase creates a new database with given name and type. +func CreateDatabase(name string, storageType string) error { + databasesLock.Lock() + defer databasesLock.Unlock() + _, ok := databases[name] + if ok { + return errors.New("database with this name already loaded.") + } + if databaseExists(name) { + return errors.New("database with this name already exists.") + } + + iface, err := startDatabase(name) + if err != nil { + return err + } + databases[name] = iface + return nil +} + +// InjectDatabase injects an already running database into the system. +func InjectDatabase(name string, iface *storage.Interface) error { + databasesLock.Lock() + defer databasesLock.Unlock() + _, ok := databases[name] + if ok { + return errors.New("database with this name already loaded.") + } + if databaseExists(name) { + return errors.New("database with this name already exists.") + } + databases[name] = iface + return nil +} diff --git a/database/dbutils/wrapper.go b/database/dbutils/wrapper.go deleted file mode 100644 index 98ef68c..0000000 --- a/database/dbutils/wrapper.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file. - -/* -Package dbutils provides important function for datastore backends without creating an import loop. -*/ -package dbutils - -import ( - "errors" - "fmt" - - "github.com/ipfs/go-datastore" - - "github.com/Safing/safing-core/formats/dsd" - "github.com/Safing/safing-core/formats/varint" -) - -type Wrapper struct { - dbKey *datastore.Key - meta *Meta - Format uint8 - Data []byte -} - -func NewWrapper(key *datastore.Key, data []byte) (*Wrapper, error) { - // line crashes with: panic: runtime error: index out of range - format, _, err := varint.Unpack8(data) - if err != nil { - return nil, fmt.Errorf("database: could not get dsd format: %s", err) - } - - new := &Wrapper{ - Format: format, - Data: data, - } - new.SetKey(key) - - return new, nil -} - -func (w *Wrapper) SetKey(key *datastore.Key) { - w.dbKey = key -} - -func (w *Wrapper) GetKey() *datastore.Key { - return w.dbKey -} - -func (w *Wrapper) FmtKey() string { - return w.dbKey.String() -} - -func DumpModel(uncertain interface{}, storageType uint8) ([]byte, error) { - wrapped, ok := uncertain.(*Wrapper) - if ok { - if storageType != dsd.AUTO && storageType != wrapped.Format { - return nil, errors.New("could not dump model, wrapped object format mismatch") - } - return wrapped.Data, nil - } - - dumped, err := dsd.Dump(uncertain, storageType) - if err != nil { - return nil, err - } - return dumped, nil -} diff --git a/database/interface.go b/database/interface.go new file mode 100644 index 0000000..2d249ba --- /dev/null +++ b/database/interface.go @@ -0,0 +1,17 @@ +package database + +type Interface struct { + local bool + internal bool +} + +func NewInterface(local bool, internal bool) *Interface { + return &Interface{ + local: local, + internal: internal, + } +} + +func (i *Interface) Get(string key) (model.Model, error) { + return nil, nil +} diff --git a/database/model.go b/database/model.go index 4283ba2..2644043 100644 --- a/database/model.go +++ b/database/model.go @@ -7,8 +7,6 @@ import ( "strings" "sync" - "github.com/ipfs/go-datastore" - "github.com/Safing/safing-core/database/dbutils" "github.com/Safing/safing-core/formats/dsd" ) diff --git a/database/model/base.go b/database/model/base.go new file mode 100644 index 0000000..5ad252d --- /dev/null +++ b/database/model/base.go @@ -0,0 +1,47 @@ +package model + +import ( + "github.com/Safing/portbase/formats/dsd" +) + +// Base provides a quick way to comply with the Model interface. +type Base struct { + dbName string + dbKey string + meta *Meta +} + +// Key returns the key of the database record. +func (b *Base) Key() string { + return b.dbKey +} + +// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key. +func (b *Base) SetKey(key string) { + b.dbKey = key +} + +// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status. +func (b *Base) MoveTo(key string) { + b.dbKey = key + b.meta.Reset() +} + +// Meta returns the metadata object for this record. +func (b *Base) Meta() *Meta { + return b.meta +} + +// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key. +func (b *Base) SetMeta(meta *Meta) { + b.meta = meta +} + +// Marshal marshals the object, without the database key or metadata +func (b *Base) Marshal(format uint8) ([]byte, error) { + dumped, err := dsd.Dump(b, format) + if err != nil { + return nil, err + } + return dumped, nil +} diff --git a/database/model/formats.go b/database/model/formats.go new file mode 100644 index 0000000..6718593 --- /dev/null +++ b/database/model/formats.go @@ -0,0 +1,15 @@ +package model + +import ( + "github.com/Safing/portbase/formats/dsd" +) + +// Reimport DSD storage types +const ( + AUTO = dsd.AUTO + STRING = dsd.STRING // S + BYTES = dsd.BYTES // X + JSON = dsd.JSON // J + BSON = dsd.BSON // B + GenCode = dsd.GenCode // G (reserved) +) diff --git a/database/model/key.go b/database/model/key.go new file mode 100644 index 0000000..19c7c59 --- /dev/null +++ b/database/model/key.go @@ -0,0 +1,14 @@ +package model + +import ( + "errors" + "strings" +) + +func ParseKey(key string) (dbName, dbKey string, err error) { + splitted := strings.SplitN(key, ":", 2) + if len(splitted) == 2 { + return splitted[0], splitted[1], nil + } + return "", "", errors.New("invalid key") +} diff --git a/database/model/meta-bench_test.go b/database/model/meta-bench_test.go new file mode 100644 index 0000000..262aaec --- /dev/null +++ b/database/model/meta-bench_test.go @@ -0,0 +1,466 @@ +package model + +// Benchmark: +// BenchmarkAllocateBytes-8 2000000000 0.76 ns/op +// BenchmarkAllocateStruct1-8 2000000000 0.76 ns/op +// BenchmarkAllocateStruct2-8 2000000000 0.79 ns/op +// BenchmarkMetaSerializeContainer-8 1000000 1703 ns/op +// BenchmarkMetaUnserializeContainer-8 2000000 950 ns/op +// BenchmarkMetaSerializeVarInt-8 3000000 457 ns/op +// BenchmarkMetaUnserializeVarInt-8 20000000 62.9 ns/op +// BenchmarkMetaSerializeWithXDR2-8 1000000 2360 ns/op +// BenchmarkMetaUnserializeWithXDR2-8 500000 3189 ns/op +// BenchmarkMetaSerializeWithColfer-8 10000000 237 ns/op +// BenchmarkMetaUnserializeWithColfer-8 20000000 51.7 ns/op +// BenchmarkMetaSerializeWithCodegen-8 50000000 23.7 ns/op +// BenchmarkMetaUnserializeWithCodegen-8 100000000 18.9 ns/op +// BenchmarkMetaSerializeWithDSDJSON-8 1000000 2398 ns/op +// BenchmarkMetaUnserializeWithDSDJSON-8 300000 6264 ns/op + +import ( + "testing" + "time" + + "github.com/Safing/portbase/container" + "github.com/Safing/portbase/formats/dsd" + "github.com/Safing/portbase/formats/varint" + // Colfer + // "github.com/Safing/portbase/database/model/model" + // XDR + // xdr2 "github.com/davecgh/go-xdr/xdr2" +) + +var ( + testMeta = &Meta{ + created: time.Now().Unix(), + modified: time.Now().Unix(), + expires: time.Now().Unix(), + deleted: time.Now().Unix(), + secret: true, + cronjewel: true, + } +) + +func BenchmarkAllocateBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = make([]byte, 33) + } +} + +func BenchmarkAllocateStruct1(b *testing.B) { + for i := 0; i < b.N; i++ { + var new Meta + _ = new + } +} + +func BenchmarkAllocateStruct2(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = Meta{} + } +} + +func BenchmarkMetaSerializeContainer(b *testing.B) { + + // Start benchmark + for i := 0; i < b.N; i++ { + c := container.New() + c.AppendNumber(uint64(testMeta.created)) + c.AppendNumber(uint64(testMeta.modified)) + c.AppendNumber(uint64(testMeta.expires)) + c.AppendNumber(uint64(testMeta.deleted)) + switch { + case testMeta.secret && testMeta.cronjewel: + c.AppendNumber(3) + case testMeta.secret: + c.AppendNumber(1) + case testMeta.cronjewel: + c.AppendNumber(2) + default: + c.AppendNumber(0) + } + } + +} + +func BenchmarkMetaUnserializeContainer(b *testing.B) { + + // Setup + c := container.New() + c.AppendNumber(uint64(testMeta.created)) + c.AppendNumber(uint64(testMeta.modified)) + c.AppendNumber(uint64(testMeta.expires)) + c.AppendNumber(uint64(testMeta.deleted)) + switch { + case testMeta.secret && testMeta.cronjewel: + c.AppendNumber(3) + case testMeta.secret: + c.AppendNumber(1) + case testMeta.cronjewel: + c.AppendNumber(2) + default: + c.AppendNumber(0) + } + encodedData := c.CompileData() + + // Reset timer for precise results + b.ResetTimer() + + // Start benchmark + for i := 0; i < b.N; i++ { + var newMeta Meta + var err error + var num uint64 + c := container.New(encodedData) + num, err = c.GetNextN64() + newMeta.created = int64(num) + if err != nil { + b.Errorf("could not decode: %s", err) + return + } + num, err = c.GetNextN64() + newMeta.modified = int64(num) + if err != nil { + b.Errorf("could not decode: %s", err) + return + } + num, err = c.GetNextN64() + newMeta.expires = int64(num) + if err != nil { + b.Errorf("could not decode: %s", err) + return + } + num, err = c.GetNextN64() + newMeta.deleted = int64(num) + if err != nil { + b.Errorf("could not decode: %s", err) + return + } + + flags, err := c.GetNextN8() + if err != nil { + b.Errorf("could not decode: %s", err) + return + } + + switch flags { + case 3: + newMeta.secret = true + newMeta.cronjewel = true + case 2: + newMeta.cronjewel = true + case 1: + newMeta.secret = true + case 0: + default: + b.Errorf("invalid flag value: %d", flags) + return + } + } + +} + +func BenchmarkMetaSerializeVarInt(b *testing.B) { + + // Start benchmark + for i := 0; i < b.N; i++ { + encoded := make([]byte, 33) + offset := 0 + data := varint.Pack64(uint64(testMeta.created)) + for _, part := range data { + encoded[offset] = part + offset++ + } + data = varint.Pack64(uint64(testMeta.modified)) + for _, part := range data { + encoded[offset] = part + offset++ + } + data = varint.Pack64(uint64(testMeta.expires)) + for _, part := range data { + encoded[offset] = part + offset++ + } + data = varint.Pack64(uint64(testMeta.deleted)) + for _, part := range data { + encoded[offset] = part + offset++ + } + + switch { + case testMeta.secret && testMeta.cronjewel: + encoded[offset] = 3 + case testMeta.secret: + encoded[offset] = 1 + case testMeta.cronjewel: + encoded[offset] = 2 + default: + encoded[offset] = 0 + } + offset++ + } + +} + +func BenchmarkMetaUnserializeVarInt(b *testing.B) { + + // Setup + encoded := make([]byte, 33) + offset := 0 + data := varint.Pack64(uint64(testMeta.created)) + for _, part := range data { + encoded[offset] = part + offset++ + } + data = varint.Pack64(uint64(testMeta.modified)) + for _, part := range data { + encoded[offset] = part + offset++ + } + data = varint.Pack64(uint64(testMeta.expires)) + for _, part := range data { + encoded[offset] = part + offset++ + } + data = varint.Pack64(uint64(testMeta.deleted)) + for _, part := range data { + encoded[offset] = part + offset++ + } + + switch { + case testMeta.secret && testMeta.cronjewel: + encoded[offset] = 3 + case testMeta.secret: + encoded[offset] = 1 + case testMeta.cronjewel: + encoded[offset] = 2 + default: + encoded[offset] = 0 + } + offset++ + encodedData := encoded[:offset] + + // Reset timer for precise results + b.ResetTimer() + + // Start benchmark + for i := 0; i < b.N; i++ { + var newMeta Meta + offset = 0 + + num, n, err := varint.Unpack64(encodedData) + if err != nil { + b.Error(err) + return + } + testMeta.created = int64(num) + offset += n + + num, n, err = varint.Unpack64(encodedData[offset:]) + if err != nil { + b.Error(err) + return + } + testMeta.modified = int64(num) + offset += n + + num, n, err = varint.Unpack64(encodedData[offset:]) + if err != nil { + b.Error(err) + return + } + testMeta.expires = int64(num) + offset += n + + num, n, err = varint.Unpack64(encodedData[offset:]) + if err != nil { + b.Error(err) + return + } + testMeta.deleted = int64(num) + offset += n + + switch encodedData[offset] { + case 3: + newMeta.secret = true + newMeta.cronjewel = true + case 2: + newMeta.cronjewel = true + case 1: + newMeta.secret = true + case 0: + default: + b.Errorf("invalid flag value: %d", encodedData[offset]) + return + } + } + +} + +// func BenchmarkMetaSerializeWithXDR2(b *testing.B) { +// +// // Setup +// var w bytes.Buffer +// +// // Reset timer for precise results +// b.ResetTimer() +// +// // Start benchmark +// for i := 0; i < b.N; i++ { +// w.Reset() +// _, err := xdr2.Marshal(&w, testMeta) +// if err != nil { +// b.Errorf("failed to serialize with xdr2: %s", err) +// return +// } +// } +// +// } + +// func BenchmarkMetaUnserializeWithXDR2(b *testing.B) { +// +// // Setup +// var w bytes.Buffer +// _, err := xdr2.Marshal(&w, testMeta) +// if err != nil { +// b.Errorf("failed to serialize with xdr2: %s", err) +// } +// encodedData := w.Bytes() +// +// // Reset timer for precise results +// b.ResetTimer() +// +// // Start benchmark +// for i := 0; i < b.N; i++ { +// var newMeta Meta +// _, err := xdr2.Unmarshal(bytes.NewReader(encodedData), &newMeta) +// if err != nil { +// b.Errorf("failed to unserialize with xdr2: %s", err) +// return +// } +// } +// +// } + +// func BenchmarkMetaSerializeWithColfer(b *testing.B) { +// +// testColf := &model.Course{ +// Created: time.Now().Unix(), +// Modified: time.Now().Unix(), +// Expires: time.Now().Unix(), +// Deleted: time.Now().Unix(), +// Secret: true, +// Cronjewel: true, +// } +// +// // Setup +// for i := 0; i < b.N; i++ { +// _, err := testColf.MarshalBinary() +// if err != nil { +// b.Errorf("failed to serialize with colfer: %s", err) +// return +// } +// } +// +// } + +// func BenchmarkMetaUnserializeWithColfer(b *testing.B) { +// +// testColf := &model.Course{ +// Created: time.Now().Unix(), +// Modified: time.Now().Unix(), +// Expires: time.Now().Unix(), +// Deleted: time.Now().Unix(), +// Secret: true, +// Cronjewel: true, +// } +// encodedData, err := testColf.MarshalBinary() +// if err != nil { +// b.Errorf("failed to serialize with colfer: %s", err) +// return +// } +// +// // Setup +// for i := 0; i < b.N; i++ { +// var testUnColf model.Course +// err := testUnColf.UnmarshalBinary(encodedData) +// if err != nil { +// b.Errorf("failed to unserialize with colfer: %s", err) +// return +// } +// } +// +// } + +func BenchmarkMetaSerializeWithCodegen(b *testing.B) { + + for i := 0; i < b.N; i++ { + _, err := testMeta.GenCodeMarshal(nil) + if err != nil { + b.Errorf("failed to serialize with codegen: %s", err) + return + } + } + +} + +func BenchmarkMetaUnserializeWithCodegen(b *testing.B) { + + // Setup + encodedData, err := testMeta.GenCodeMarshal(nil) + if err != nil { + b.Errorf("failed to serialize with codegen: %s", err) + return + } + + // Reset timer for precise results + b.ResetTimer() + + // Start benchmark + for i := 0; i < b.N; i++ { + var newMeta Meta + _, err := newMeta.GenCodeUnmarshal(encodedData) + if err != nil { + b.Errorf("failed to unserialize with codegen: %s", err) + return + } + } + +} + +func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) { + + for i := 0; i < b.N; i++ { + _, err := dsd.Dump(testMeta, dsd.JSON) + if err != nil { + b.Errorf("failed to serialize with DSD/JSON: %s", err) + return + } + } + +} + +func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) { + + // Setup + encodedData, err := dsd.Dump(testMeta, dsd.JSON) + if err != nil { + b.Errorf("failed to serialize with DSD/JSON: %s", err) + return + } + + // Reset timer for precise results + b.ResetTimer() + + // Start benchmark + for i := 0; i < b.N; i++ { + var newMeta Meta + _, err := dsd.Load(encodedData, &newMeta) + if err != nil { + b.Errorf("failed to unserialize with DSD/JSON: %s", err) + return + } + } + +} diff --git a/database/model/meta-gencode.go b/database/model/meta-gencode.go new file mode 100644 index 0000000..6c4e76f --- /dev/null +++ b/database/model/meta-gencode.go @@ -0,0 +1,157 @@ +package model + +import ( + "io" + "time" + "unsafe" +) + +var ( + _ = unsafe.Sizeof(0) + _ = io.ReadFull + _ = time.Now() +) + +// GenCodeSize returns the size of the gencode marshalled byte slice +func (d *Meta) GenCodeSize() (s uint64) { + s += 34 + return +} + +// GenCodeMarshal gencode marshalls Meta into the given byte array, or a new one if its too small. +func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) { + size := d.GenCodeSize() + { + if uint64(cap(buf)) >= size { + buf = buf[:size] + } else { + buf = make([]byte, size) + } + } + i := uint64(0) + + { + + buf[0+0] = byte(d.created >> 0) + + buf[1+0] = byte(d.created >> 8) + + buf[2+0] = byte(d.created >> 16) + + buf[3+0] = byte(d.created >> 24) + + buf[4+0] = byte(d.created >> 32) + + buf[5+0] = byte(d.created >> 40) + + buf[6+0] = byte(d.created >> 48) + + buf[7+0] = byte(d.created >> 56) + + } + { + + buf[0+8] = byte(d.modified >> 0) + + buf[1+8] = byte(d.modified >> 8) + + buf[2+8] = byte(d.modified >> 16) + + buf[3+8] = byte(d.modified >> 24) + + buf[4+8] = byte(d.modified >> 32) + + buf[5+8] = byte(d.modified >> 40) + + buf[6+8] = byte(d.modified >> 48) + + buf[7+8] = byte(d.modified >> 56) + + } + { + + buf[0+16] = byte(d.expires >> 0) + + buf[1+16] = byte(d.expires >> 8) + + buf[2+16] = byte(d.expires >> 16) + + buf[3+16] = byte(d.expires >> 24) + + buf[4+16] = byte(d.expires >> 32) + + buf[5+16] = byte(d.expires >> 40) + + buf[6+16] = byte(d.expires >> 48) + + buf[7+16] = byte(d.expires >> 56) + + } + { + + buf[0+24] = byte(d.deleted >> 0) + + buf[1+24] = byte(d.deleted >> 8) + + buf[2+24] = byte(d.deleted >> 16) + + buf[3+24] = byte(d.deleted >> 24) + + buf[4+24] = byte(d.deleted >> 32) + + buf[5+24] = byte(d.deleted >> 40) + + buf[6+24] = byte(d.deleted >> 48) + + buf[7+24] = byte(d.deleted >> 56) + + } + { + if d.secret { + buf[32] = 1 + } else { + buf[32] = 0 + } + } + { + if d.cronjewel { + buf[33] = 1 + } else { + buf[33] = 0 + } + } + return buf[:i+34], nil +} + +// GenCodeUnmarshal gencode unmarshalls Meta and returns the bytes read. +func (d *Meta) GenCodeUnmarshal(buf []byte) (uint64, error) { + i := uint64(0) + + { + + d.created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56) + + } + { + + d.modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56) + + } + { + + d.expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56) + + } + { + + d.deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56) + + } + { + d.secret = buf[32] == 1 + } + { + d.cronjewel = buf[33] == 1 + } + return i + 34, nil +} diff --git a/database/model/meta-gencode_test.go b/database/model/meta-gencode_test.go new file mode 100644 index 0000000..4f3bdcb --- /dev/null +++ b/database/model/meta-gencode_test.go @@ -0,0 +1,35 @@ +package model + +import ( + "reflect" + "testing" + "time" +) + +var ( + genCodeTestMeta = &Meta{ + created: time.Now().Unix(), + modified: time.Now().Unix(), + expires: time.Now().Unix(), + deleted: time.Now().Unix(), + secret: true, + cronjewel: true, + } +) + +func TestGenCode(t *testing.T) { + encoded, err := genCodeTestMeta.GenCodeMarshal(nil) + if err != nil { + t.Fatal(err) + } + + new := &Meta{} + _, err = new.GenCodeUnmarshal(encoded) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(genCodeTestMeta, new) { + t.Errorf("objects are not equal, got: %v", new) + } +} diff --git a/database/model/meta.gencode.gen.go b/database/model/meta.gencode.gen.go deleted file mode 100644 index 4cf07f0..0000000 --- a/database/model/meta.gencode.gen.go +++ /dev/null @@ -1,163 +0,0 @@ -package model - -import ( - "io" - "time" - "unsafe" -) - -var ( - _ = unsafe.Sizeof(0) - _ = io.ReadFull - _ = time.Now() -) - -// type Meta struct { -// Created int64 -// Modified int64 -// Expires int64 -// Deleted int64 -// Secret bool -// Cronjewel bool -// } - -func (d *Meta) Size() (s uint64) { - - s += 34 - return -} -func (d *Meta) Marshal(buf []byte) ([]byte, error) { - size := d.Size() - { - if uint64(cap(buf)) >= size { - buf = buf[:size] - } else { - buf = make([]byte, size) - } - } - i := uint64(0) - - { - - buf[0+0] = byte(d.Created >> 0) - - buf[1+0] = byte(d.Created >> 8) - - buf[2+0] = byte(d.Created >> 16) - - buf[3+0] = byte(d.Created >> 24) - - buf[4+0] = byte(d.Created >> 32) - - buf[5+0] = byte(d.Created >> 40) - - buf[6+0] = byte(d.Created >> 48) - - buf[7+0] = byte(d.Created >> 56) - - } - { - - buf[0+8] = byte(d.Modified >> 0) - - buf[1+8] = byte(d.Modified >> 8) - - buf[2+8] = byte(d.Modified >> 16) - - buf[3+8] = byte(d.Modified >> 24) - - buf[4+8] = byte(d.Modified >> 32) - - buf[5+8] = byte(d.Modified >> 40) - - buf[6+8] = byte(d.Modified >> 48) - - buf[7+8] = byte(d.Modified >> 56) - - } - { - - buf[0+16] = byte(d.Expires >> 0) - - buf[1+16] = byte(d.Expires >> 8) - - buf[2+16] = byte(d.Expires >> 16) - - buf[3+16] = byte(d.Expires >> 24) - - buf[4+16] = byte(d.Expires >> 32) - - buf[5+16] = byte(d.Expires >> 40) - - buf[6+16] = byte(d.Expires >> 48) - - buf[7+16] = byte(d.Expires >> 56) - - } - { - - buf[0+24] = byte(d.Deleted >> 0) - - buf[1+24] = byte(d.Deleted >> 8) - - buf[2+24] = byte(d.Deleted >> 16) - - buf[3+24] = byte(d.Deleted >> 24) - - buf[4+24] = byte(d.Deleted >> 32) - - buf[5+24] = byte(d.Deleted >> 40) - - buf[6+24] = byte(d.Deleted >> 48) - - buf[7+24] = byte(d.Deleted >> 56) - - } - { - if d.Secret { - buf[32] = 1 - } else { - buf[32] = 0 - } - } - { - if d.Cronjewel { - buf[33] = 1 - } else { - buf[33] = 0 - } - } - return buf[:i+34], nil -} - -func (d *Meta) Unmarshal(buf []byte) (uint64, error) { - i := uint64(0) - - { - - d.Created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56) - - } - { - - d.Modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56) - - } - { - - d.Expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56) - - } - { - - d.Deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56) - - } - { - d.Secret = buf[32] == 1 - } - { - d.Cronjewel = buf[33] == 1 - } - return i + 34, nil -} diff --git a/database/model/meta.go b/database/model/meta.go index 161ee64..2424821 100644 --- a/database/model/meta.go +++ b/database/model/meta.go @@ -1,10 +1,72 @@ package model +import "time" + +// Meta holds type Meta struct { - Created int64 `json:"c,omitempty" bson:"c,omitempty"` - Modified int64 `json:"m,omitempty" bson:"m,omitempty"` - Expires int64 `json:"e,omitempty" bson:"e,omitempty"` - Deleted int64 `json:"d,omitempty" bson:"d,omitempty"` - Secret bool `json:"s,omitempty" bson:"s,omitempty"` // secrets must not be sent to the UI, only synced between nodes - Cronjewel bool `json:"j,omitempty" bson:"j,omitempty"` // crownjewels must never leave the instance, but may be read by the UI + created int64 + modified int64 + expires int64 + deleted int64 + secret bool // secrets must not be sent to the UI, only synced between nodes + cronjewel bool // crownjewels must never leave the instance, but may be read by the UI +} + +// SetAbsoluteExpiry sets an absolute expiry time, that is not affected when the record is updated. +func (m *Meta) SetAbsoluteExpiry(time int64) { + m.expires = time + m.deleted = 0 +} + +// SetRelativateExpiry sets a relative expiry that is automatically updated whenever the record is updated/saved. +func (m *Meta) SetRelativateExpiry(duration int64) { + if duration >= 0 { + m.deleted = -duration + } +} + +// MakeCrownJewel marks the database records as a crownjewel, meaning that it will not be sent/synced to other devices. +func (m *Meta) MakeCrownJewel() { + m.cronjewel = true +} + +// MakeSecret sets the database record as secret, meaning that it may only be used internally, and not by interfacing processes, such as the UI. +func (m *Meta) MakeSecret() { + m.secret = true +} + +// Update updates the internal meta states and should be called before writing the record to the database. +func (m *Meta) Update() { + now := time.Now().Unix() + m.modified = now + if m.created == 0 { + m.created = now + } + if m.deleted < 0 { + m.expires = now - m.deleted + } +} + +// Reset resets all metadata, except for the secret and crownjewel status. +func (m *Meta) Reset() { + m.created = 0 + m.modified = 0 + m.expires = 0 + m.deleted = 0 +} + +// CheckScope checks whether the current database record exists for the given scope. +func (m *Meta) CheckScope(now int64, local, internal bool) (recordExists bool) { + switch { + case m.deleted > 0: + return false + case m.expires < now: + return false + case !local && m.cronjewel: + return false + case !internal && m.secret: + return false + default: + return true + } } diff --git a/database/model/meta_test.go b/database/model/meta_test.go deleted file mode 100644 index 4141fce..0000000 --- a/database/model/meta_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package model - -import ( - "bytes" - "testing" - "time" - - "github.com/Safing/portbase/container" - "github.com/Safing/portbase/database/model/model" - "github.com/Safing/portbase/formats/dsd" - xdr2 "github.com/davecgh/go-xdr/xdr2" -) - -var ( - testMeta = &Meta{ - Created: time.Now().Unix(), - Modified: time.Now().Unix(), - Expires: time.Now().Unix(), - Deleted: time.Now().Unix(), - Secret: true, - Cronjewel: true, - } -) - -func BenchmarkAllocateBytes(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = make([]byte, 33) - } -} - -func BenchmarkAllocateStruct1(b *testing.B) { - for i := 0; i < b.N; i++ { - var new Meta - _ = new - } -} - -func BenchmarkAllocateStruct2(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = Meta{} - } -} - -func BenchmarkMetaSerializeCustom(b *testing.B) { - - // Start benchmark - for i := 0; i < b.N; i++ { - c := container.New() - c.AppendNumber(uint64(testMeta.Created)) - c.AppendNumber(uint64(testMeta.Modified)) - c.AppendNumber(uint64(testMeta.Expires)) - c.AppendNumber(uint64(testMeta.Deleted)) - switch { - case testMeta.Secret && testMeta.Cronjewel: - c.AppendNumber(3) - case testMeta.Secret: - c.AppendNumber(1) - case testMeta.Cronjewel: - c.AppendNumber(2) - default: - c.AppendNumber(0) - } - } - -} - -func BenchmarkMetaUnserializeCustom(b *testing.B) { - - // Setup - c := container.New() - c.AppendNumber(uint64(testMeta.Created)) - c.AppendNumber(uint64(testMeta.Modified)) - c.AppendNumber(uint64(testMeta.Expires)) - c.AppendNumber(uint64(testMeta.Deleted)) - switch { - case testMeta.Secret && testMeta.Cronjewel: - c.AppendNumber(3) - case testMeta.Secret: - c.AppendNumber(1) - case testMeta.Cronjewel: - c.AppendNumber(2) - default: - c.AppendNumber(0) - } - encodedData := c.CompileData() - - // Reset timer for precise results - b.ResetTimer() - - // Start benchmark - for i := 0; i < b.N; i++ { - var newMeta Meta - var err error - var num uint64 - c := container.New(encodedData) - num, err = c.GetNextN64() - newMeta.Created = int64(num) - if err != nil { - b.Errorf("could not decode: %s", err) - return - } - num, err = c.GetNextN64() - newMeta.Modified = int64(num) - if err != nil { - b.Errorf("could not decode: %s", err) - return - } - num, err = c.GetNextN64() - newMeta.Expires = int64(num) - if err != nil { - b.Errorf("could not decode: %s", err) - return - } - num, err = c.GetNextN64() - newMeta.Deleted = int64(num) - if err != nil { - b.Errorf("could not decode: %s", err) - return - } - - flags, err := c.GetNextN8() - if err != nil { - b.Errorf("could not decode: %s", err) - return - } - - switch flags { - case 3: - newMeta.Secret = true - newMeta.Cronjewel = true - case 2: - newMeta.Cronjewel = true - case 1: - newMeta.Secret = true - case 0: - default: - b.Errorf("invalid flag value: %d", flags) - return - } - } - -} - -func BenchmarkMetaSerializeWithXDR2(b *testing.B) { - - // Setup - var w bytes.Buffer - - // Reset timer for precise results - b.ResetTimer() - - // Start benchmark - for i := 0; i < b.N; i++ { - w.Reset() - _, err := xdr2.Marshal(&w, testMeta) - if err != nil { - b.Errorf("failed to serialize with xdr2: %s", err) - return - } - } - -} - -func BenchmarkMetaUnserializeWithXDR2(b *testing.B) { - - // Setup - var w bytes.Buffer - _, err := xdr2.Marshal(&w, testMeta) - if err != nil { - b.Errorf("failed to serialize with xdr2: %s", err) - } - encodedData := w.Bytes() - - // Reset timer for precise results - b.ResetTimer() - - // Start benchmark - for i := 0; i < b.N; i++ { - var newMeta Meta - _, err := xdr2.Unmarshal(bytes.NewReader(encodedData), &newMeta) - if err != nil { - b.Errorf("failed to unserialize with xdr2: %s", err) - return - } - } - -} - -func BenchmarkMetaSerializeWithColfer(b *testing.B) { - - testColf := &model.Course{ - Created: time.Now().Unix(), - Modified: time.Now().Unix(), - Expires: time.Now().Unix(), - Deleted: time.Now().Unix(), - Secret: true, - Cronjewel: true, - } - - // Setup - for i := 0; i < b.N; i++ { - _, err := testColf.MarshalBinary() - if err != nil { - b.Errorf("failed to serialize with colfer: %s", err) - return - } - } - -} - -func BenchmarkMetaUnserializeWithColfer(b *testing.B) { - - testColf := &model.Course{ - Created: time.Now().Unix(), - Modified: time.Now().Unix(), - Expires: time.Now().Unix(), - Deleted: time.Now().Unix(), - Secret: true, - Cronjewel: true, - } - encodedData, err := testColf.MarshalBinary() - if err != nil { - b.Errorf("failed to serialize with colfer: %s", err) - return - } - - // Setup - for i := 0; i < b.N; i++ { - var testUnColf model.Course - err := testUnColf.UnmarshalBinary(encodedData) - if err != nil { - b.Errorf("failed to unserialize with colfer: %s", err) - return - } - } - -} - -func BenchmarkMetaSerializeWithCodegen(b *testing.B) { - - for i := 0; i < b.N; i++ { - buf := make([]byte, 34) - _, err := testMeta.Marshal(buf) - if err != nil { - b.Errorf("failed to serialize with codegen: %s", err) - return - } - } - -} - -func BenchmarkMetaUnserializeWithCodegen(b *testing.B) { - - // Setup - buf := make([]byte, 34) - encodedData, err := testMeta.Marshal(buf) - if err != nil { - b.Errorf("failed to serialize with codegen: %s", err) - return - } - - // Reset timer for precise results - b.ResetTimer() - - // Start benchmark - for i := 0; i < b.N; i++ { - var newMeta Meta - _, err := newMeta.Unmarshal(encodedData) - if err != nil { - b.Errorf("failed to unserialize with codegen: %s", err) - return - } - } - -} - -func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) { - - for i := 0; i < b.N; i++ { - _, err := dsd.Dump(testMeta, dsd.JSON) - if err != nil { - b.Errorf("failed to serialize with DSD/JSON: %s", err) - return - } - } - -} - -func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) { - - // Setup - encodedData, err := dsd.Dump(testMeta, dsd.JSON) - if err != nil { - b.Errorf("failed to serialize with DSD/JSON: %s", err) - return - } - - // Reset timer for precise results - b.ResetTimer() - - // Start benchmark - for i := 0; i < b.N; i++ { - var newMeta Meta - _, err := dsd.Load(encodedData, &newMeta) - if err != nil { - b.Errorf("failed to unserialize with DSD/JSON: %s", err) - return - } - } - -} diff --git a/database/model/model.go b/database/model/model.go index 93debc9..951f6f3 100644 --- a/database/model/model.go +++ b/database/model/model.go @@ -1,4 +1,11 @@ package model +// Model provides an interface for uniformally handling database records. type Model interface { + Key() string + SetKey(key string) + MoveTo(key string) + Meta() *Meta + SetMeta(meta *Meta) + Marshal(format uint8) ([]byte, error) } diff --git a/database/model/wrapper.go b/database/model/wrapper.go new file mode 100644 index 0000000..a19f9d1 --- /dev/null +++ b/database/model/wrapper.go @@ -0,0 +1,67 @@ +package model + +import ( + "errors" + "fmt" + + "github.com/Safing/safing-core/formats/dsd" + "github.com/Safing/safing-core/formats/varint" +) + +type Wrapper struct { + dbName string + dbKey string + meta *Meta + Format uint8 + Data []byte +} + +func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) { + format, _, err := varint.Unpack8(data) + if err != nil { + return nil, fmt.Errorf("database: could not get dsd format: %s", err) + } + + new := &Wrapper{ + dbKey: key, + meta: meta, + Format: format, + Data: data, + } + + return new, nil +} + +// Key returns the key of the database record. +func (w *Wrapper) Key() string { + return w.dbKey +} + +// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key. +func (w *Wrapper) SetKey(key string) { + w.dbKey = key +} + +// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status. +func (w *Wrapper) MoveTo(key string) { + w.dbKey = key + w.meta.Reset() +} + +// Meta returns the metadata object for this record. +func (w *Wrapper) Meta() *Meta { + return w.meta +} + +// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key. +func (w *Wrapper) SetMeta(meta *Meta) { + w.meta = meta +} + +// Marshal marshals the object, without the database key or metadata +func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) { + if storageType != dsd.AUTO && storageType != w.Format { + return nil, errors.New("could not dump model, wrapped object format mismatch") + } + return w.Data, nil +} diff --git a/database/storage/interface.go b/database/storage/interface.go index 108a25f..e1fa952 100644 --- a/database/storage/interface.go +++ b/database/storage/interface.go @@ -1,30 +1,34 @@ package storage import ( - "github.com/Safing/portbase/database/iterator" - "github.com/Safing/portbase/database/model" + "github.com/Safing/portbase/database/iterator" + "github.com/Safing/portbase/database/model" + "github.com/Safing/portbase/database/query" ) // Interface defines the database storage API. type Interface interface { - // Full - Exists(key string) (bool, error) - Get(key string) (model.Model, error) - Create(key string, model model.Model) error - Update(key string, model model.Model) error // create when not exists - UpdateOrCreate(key string, model model.Model) error // update, create if not exists. - Delete(key string) error + // Retrieve + Exists(key string) (bool, error) + Get(key string) (model.Model, error) - // Partial - // What happens if I mutate a value that does not yet exist? How would I know its type? - InsertPartial(key string, partialObject interface{}) - InsertValue(key string, attribute string, value interface{}) + // Modify + Create(model model.Model) error + Update(model model.Model) error // create when not exists + UpdateOrCreate(model model.Model) error // update, create if not exists. + Delete(key string) error - // Query - Query(*query.Query) (*iterator.Iterator, error) + // Partial + // What happens if I mutate a value that does not yet exist? How would I know its type? + InsertPartial(key string, partialObject interface{}) + InsertValue(key string, attribute string, value interface{}) - // Meta - LetExpire(key string, timestamp int64) error - MakeSecret(key string) error // only visible internal - MakeCrownJewel(key string) error // do not sync between devices + // Query + Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) + + // Meta + SetAbsoluteExpiry(key string, time int64) + SetRelativateExpiry(key string, duration int64) + MakeCrownJewel(key string) + MakeSecret(key string) } diff --git a/database/storages.go b/database/storages.go new file mode 100644 index 0000000..8e841f6 --- /dev/null +++ b/database/storages.go @@ -0,0 +1,36 @@ +package database + +// A Factory creates a new database of it's type. +type Factory func(name, location string) (*storage.Interface, error) + +var ( + storages map[string]Factory + storagesLock sync.Mutex +) + +// RegisterStorage registers a new storage type. +func RegisterStorage(name string, factory Factory) error { + storagesLock.Lock() + defer storagesLock.Unlock() + + _, ok := storages[name] + if ok { + return errors.New("factory for this type already exists") + } + + storages[name] = factory + return nil +} + +// startDatabase starts a new database with the given name, storageType at location. +func startDatabase(name, storageType, location string) (*storage.Interface, error) { + storagesLock.Lock() + defer storagesLock.Unlock() + + factory, ok := storages[name] + if !ok { + return fmt.Errorf("storage of this type (%s) does not exist", storageType) + } + + return factory(name, location) +}