vikarti.anatra/safing-portbase
1
0
Fork 0
mirror of https://github.com/safing/portbase synced 2025-09-01 01:59:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Work on database revamp [WIP]

Browse source
This commit is contained in:
Daniel 2018-09-04 17:09:32 +02:00
parent 307ddd70fd
commit efabb291d7
19 changed files with 1032 additions and 664 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
database/base.go
View file

@ -1,63 +0,0 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package database
import (
"errors"
"strings"
"github.com/Safing/safing-core/database/dbutils"
"github.com/ipfs/go-datastore"
uuid "github.com/satori/go.uuid"
)
type Base struct {
dbKey *datastore.Key
meta *dbutils.Meta
}
func (m *Base) SetKey(key *datastore.Key) {
m.dbKey = key
}
func (m *Base) GetKey() *datastore.Key {
return m.dbKey
}
func (m *Base) FmtKey() string {
return m.dbKey.String()
}
func (m *Base) Meta() *dbutils.Meta {
return m.meta
}
func (m *Base) CreateObject(namespace *datastore.Key, name string, model Model) error {
var newKey datastore.Key
if name == "" {
newKey = NewInstance(namespace.ChildString(getTypeName(model)), strings.Replace(uuid.NewV4().String(), "-", "", -1))
} else {
newKey = NewInstance(namespace.ChildString(getTypeName(model)), name)
}
m.dbKey = &newKey
return Create(*m.dbKey, model)
}
func (m *Base) SaveObject(model Model) error {
if m.dbKey == nil {
return errors.New("cannot save new object, use Create() instead")
}
return Update(*m.dbKey, model)
}
func (m *Base) Delete() error {
if m.dbKey == nil {
return errors.New("cannot delete object unsaved object")
}
return Delete(*m.dbKey)
}
func NewInstance(k datastore.Key, s string) datastore.Key {
return datastore.NewKey(k.String() + ":" + s)
}

57
database/database.go
View file

@ -9,29 +9,20 @@ import (
"path" "path"
"strings" "strings"
ds "github.com/ipfs/go-datastore"
dsq "github.com/ipfs/go-datastore/query"
mount "github.com/ipfs/go-datastore/syncmount"
"github.com/Safing/safing-core/database/dbutils" "github.com/Safing/safing-core/database/dbutils"
"github.com/Safing/safing-core/database/ds/channelshim"
"github.com/Safing/safing-core/database/ds/leveldb"
"github.com/Safing/safing-core/log" "github.com/Safing/safing-core/log"
"github.com/Safing/safing-core/meta" "github.com/Safing/safing-core/meta"
) )
// TODO: do not let other modules panic, even if database module crashes.
var db ds.Datastore
var ErrNotFound = errors.New("database: entry could not be found") var ErrNotFound = errors.New("database: entry could not be found")
func init() { func init() {
if strings.HasSuffix(os.Args[0], ".test") { // if strings.HasSuffix(os.Args[0], ".test") {
// testing setup // // testing setup
log.Warning("===== DATABASE RUNNING IN TEST MODE =====") // log.Warning("===== DATABASE RUNNING IN TEST MODE =====")
db = channelshim.NewChanneledDatastore(ds.NewMapDatastore()) // db = channelshim.NewChanneledDatastore(ds.NewMapDatastore())
return // return
} // }
// sfsDB, err := simplefs.NewDatastore(meta.DatabaseDir()) // sfsDB, err := simplefs.NewDatastore(meta.DatabaseDir())
// if err != nil { // if err != nil {
@ -39,24 +30,24 @@ func init() {
// os.Exit(1) // os.Exit(1)
// } // }
ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{}) // ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{})
if err != nil { // if err != nil {
fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err) // fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err)
os.Exit(1) // os.Exit(1)
} // }
//
mapDB := ds.NewMapDatastore() // mapDB := ds.NewMapDatastore()
//
db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{ // db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{
mount.Mount{ // mount.Mount{
Prefix: ds.NewKey("/Run"), // Prefix: ds.NewKey("/Run"),
Datastore: mapDB, // Datastore: mapDB,
}, // },
mount.Mount{ // mount.Mount{
Prefix: ds.NewKey("/"), // Prefix: ds.NewKey("/"),
Datastore: ldb, // Datastore: ldb,
}, // },
})) // }))
} }

56
database/databases.go Normal file
View file

@ -0,0 +1,56 @@
package database
var (
databases = make(map[string]*storage.Interface)
databasesLock sync.Mutex
)
func getDatabase(name string) *storage.Interface {
databasesLock.Lock()
defer databasesLock.Unlock()
storage, ok := databases[name]
if ok {
return
}
}
func databaseExists(name string) (exists bool) {
// check if folder exists
return true
}
// CreateDatabase creates a new database with given name and type.
func CreateDatabase(name string, storageType string) error {
databasesLock.Lock()
defer databasesLock.Unlock()
_, ok := databases[name]
if ok {
return errors.New("database with this name already loaded.")
}
if databaseExists(name) {
return errors.New("database with this name already exists.")
}
iface, err := startDatabase(name)
if err != nil {
return err
}
databases[name] = iface
return nil
}
// InjectDatabase injects an already running database into the system.
func InjectDatabase(name string, iface *storage.Interface) error {
databasesLock.Lock()
defer databasesLock.Unlock()
_, ok := databases[name]
if ok {
return errors.New("database with this name already loaded.")
}
if databaseExists(name) {
return errors.New("database with this name already exists.")
}
databases[name] = iface
return nil
}

67
database/dbutils/wrapper.go
View file

@ -1,67 +0,0 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
/*
Package dbutils provides important function for datastore backends without creating an import loop.
*/
package dbutils
import (
"errors"
"fmt"
"github.com/ipfs/go-datastore"
"github.com/Safing/safing-core/formats/dsd"
"github.com/Safing/safing-core/formats/varint"
)
type Wrapper struct {
dbKey *datastore.Key
meta *Meta
Format uint8
Data []byte
}
func NewWrapper(key *datastore.Key, data []byte) (*Wrapper, error) {
// line crashes with: panic: runtime error: index out of range
format, _, err := varint.Unpack8(data)
if err != nil {
return nil, fmt.Errorf("database: could not get dsd format: %s", err)
}
new := &Wrapper{
Format: format,
Data: data,
}
new.SetKey(key)
return new, nil
}
func (w *Wrapper) SetKey(key *datastore.Key) {
w.dbKey = key
}
func (w *Wrapper) GetKey() *datastore.Key {
return w.dbKey
}
func (w *Wrapper) FmtKey() string {
return w.dbKey.String()
}
func DumpModel(uncertain interface{}, storageType uint8) ([]byte, error) {
wrapped, ok := uncertain.(*Wrapper)
if ok {
if storageType != dsd.AUTO && storageType != wrapped.Format {
return nil, errors.New("could not dump model, wrapped object format mismatch")
}
return wrapped.Data, nil
}
dumped, err := dsd.Dump(uncertain, storageType)
if err != nil {
return nil, err
}
return dumped, nil
}

17
database/interface.go Normal file
View file

@ -0,0 +1,17 @@
package database
type Interface struct {
local bool
internal bool
}
func NewInterface(local bool, internal bool) *Interface {
return &Interface{
local: local,
internal: internal,
}
}
func (i *Interface) Get(string key) (model.Model, error) {
return nil, nil
}

2
database/model.go
View file

@ -7,8 +7,6 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/ipfs/go-datastore"
"github.com/Safing/safing-core/database/dbutils" "github.com/Safing/safing-core/database/dbutils"
"github.com/Safing/safing-core/formats/dsd" "github.com/Safing/safing-core/formats/dsd"
) )

47
database/model/base.go Normal file
View file

@ -0,0 +1,47 @@
package model
import (
"github.com/Safing/portbase/formats/dsd"
)
// Base provides a quick way to comply with the Model interface.
type Base struct {
dbName string
dbKey string
meta *Meta
}
// Key returns the key of the database record.
func (b *Base) Key() string {
return b.dbKey
}
// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
func (b *Base) SetKey(key string) {
b.dbKey = key
}
// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
func (b *Base) MoveTo(key string) {
b.dbKey = key
b.meta.Reset()
}
// Meta returns the metadata object for this record.
func (b *Base) Meta() *Meta {
return b.meta
}
// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
func (b *Base) SetMeta(meta *Meta) {
b.meta = meta
}
// Marshal marshals the object, without the database key or metadata
func (b *Base) Marshal(format uint8) ([]byte, error) {
dumped, err := dsd.Dump(b, format)
if err != nil {
return nil, err
}
return dumped, nil
}

15
database/model/formats.go Normal file
View file

@ -0,0 +1,15 @@
package model
import (
"github.com/Safing/portbase/formats/dsd"
)
// Reimport DSD storage types
const (
AUTO = dsd.AUTO
STRING = dsd.STRING // S
BYTES = dsd.BYTES // X
JSON = dsd.JSON // J
BSON = dsd.BSON // B
GenCode = dsd.GenCode // G (reserved)
)

14
database/model/key.go Normal file
View file

@ -0,0 +1,14 @@
package model
import (
"errors"
"strings"
)
func ParseKey(key string) (dbName, dbKey string, err error) {
splitted := strings.SplitN(key, ":", 2)
if len(splitted) == 2 {
return splitted[0], splitted[1], nil
}
return "", "", errors.New("invalid key")
}

466
database/model/meta-bench_test.go Normal file
View file

@ -0,0 +1,466 @@
package model
// Benchmark:
// BenchmarkAllocateBytes-8 2000000000 0.76 ns/op
// BenchmarkAllocateStruct1-8 2000000000 0.76 ns/op
// BenchmarkAllocateStruct2-8 2000000000 0.79 ns/op
// BenchmarkMetaSerializeContainer-8 1000000 1703 ns/op
// BenchmarkMetaUnserializeContainer-8 2000000 950 ns/op
// BenchmarkMetaSerializeVarInt-8 3000000 457 ns/op
// BenchmarkMetaUnserializeVarInt-8 20000000 62.9 ns/op
// BenchmarkMetaSerializeWithXDR2-8 1000000 2360 ns/op
// BenchmarkMetaUnserializeWithXDR2-8 500000 3189 ns/op
// BenchmarkMetaSerializeWithColfer-8 10000000 237 ns/op
// BenchmarkMetaUnserializeWithColfer-8 20000000 51.7 ns/op
// BenchmarkMetaSerializeWithCodegen-8 50000000 23.7 ns/op
// BenchmarkMetaUnserializeWithCodegen-8 100000000 18.9 ns/op
// BenchmarkMetaSerializeWithDSDJSON-8 1000000 2398 ns/op
// BenchmarkMetaUnserializeWithDSDJSON-8 300000 6264 ns/op
import (
"testing"
"time"
"github.com/Safing/portbase/container"
"github.com/Safing/portbase/formats/dsd"
"github.com/Safing/portbase/formats/varint"
// Colfer
// "github.com/Safing/portbase/database/model/model"
// XDR
// xdr2 "github.com/davecgh/go-xdr/xdr2"
)
var (
testMeta = &Meta{
created: time.Now().Unix(),
modified: time.Now().Unix(),
expires: time.Now().Unix(),
deleted: time.Now().Unix(),
secret: true,
cronjewel: true,
}
)
func BenchmarkAllocateBytes(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = make([]byte, 33)
}
}
func BenchmarkAllocateStruct1(b *testing.B) {
for i := 0; i < b.N; i++ {
var new Meta
_ = new
}
}
func BenchmarkAllocateStruct2(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = Meta{}
}
}
func BenchmarkMetaSerializeContainer(b *testing.B) {
// Start benchmark
for i := 0; i < b.N; i++ {
c := container.New()
c.AppendNumber(uint64(testMeta.created))
c.AppendNumber(uint64(testMeta.modified))
c.AppendNumber(uint64(testMeta.expires))
c.AppendNumber(uint64(testMeta.deleted))
switch {
case testMeta.secret && testMeta.cronjewel:
c.AppendNumber(3)
case testMeta.secret:
c.AppendNumber(1)
case testMeta.cronjewel:
c.AppendNumber(2)
default:
c.AppendNumber(0)
}
}
}
func BenchmarkMetaUnserializeContainer(b *testing.B) {
// Setup
c := container.New()
c.AppendNumber(uint64(testMeta.created))
c.AppendNumber(uint64(testMeta.modified))
c.AppendNumber(uint64(testMeta.expires))
c.AppendNumber(uint64(testMeta.deleted))
switch {
case testMeta.secret && testMeta.cronjewel:
c.AppendNumber(3)
case testMeta.secret:
c.AppendNumber(1)
case testMeta.cronjewel:
c.AppendNumber(2)
default:
c.AppendNumber(0)
}
encodedData := c.CompileData()
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
var err error
var num uint64
c := container.New(encodedData)
num, err = c.GetNextN64()
newMeta.created = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
num, err = c.GetNextN64()
newMeta.modified = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
num, err = c.GetNextN64()
newMeta.expires = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
num, err = c.GetNextN64()
newMeta.deleted = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
flags, err := c.GetNextN8()
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
switch flags {
case 3:
newMeta.secret = true
newMeta.cronjewel = true
case 2:
newMeta.cronjewel = true
case 1:
newMeta.secret = true
case 0:
default:
b.Errorf("invalid flag value: %d", flags)
return
}
}
}
func BenchmarkMetaSerializeVarInt(b *testing.B) {
// Start benchmark
for i := 0; i < b.N; i++ {
encoded := make([]byte, 33)
offset := 0
data := varint.Pack64(uint64(testMeta.created))
for _, part := range data {
encoded[offset] = part
offset++
}
data = varint.Pack64(uint64(testMeta.modified))
for _, part := range data {
encoded[offset] = part
offset++
}
data = varint.Pack64(uint64(testMeta.expires))
for _, part := range data {
encoded[offset] = part
offset++
}
data = varint.Pack64(uint64(testMeta.deleted))
for _, part := range data {
encoded[offset] = part
offset++
}
switch {
case testMeta.secret && testMeta.cronjewel:
encoded[offset] = 3
case testMeta.secret:
encoded[offset] = 1
case testMeta.cronjewel:
encoded[offset] = 2
default:
encoded[offset] = 0
}
offset++
}
}
func BenchmarkMetaUnserializeVarInt(b *testing.B) {
// Setup
encoded := make([]byte, 33)
offset := 0
data := varint.Pack64(uint64(testMeta.created))
for _, part := range data {
encoded[offset] = part
offset++
}
data = varint.Pack64(uint64(testMeta.modified))
for _, part := range data {
encoded[offset] = part
offset++
}
data = varint.Pack64(uint64(testMeta.expires))
for _, part := range data {
encoded[offset] = part
offset++
}
data = varint.Pack64(uint64(testMeta.deleted))
for _, part := range data {
encoded[offset] = part
offset++
}
switch {
case testMeta.secret && testMeta.cronjewel:
encoded[offset] = 3
case testMeta.secret:
encoded[offset] = 1
case testMeta.cronjewel:
encoded[offset] = 2
default:
encoded[offset] = 0
}
offset++
encodedData := encoded[:offset]
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
offset = 0
num, n, err := varint.Unpack64(encodedData)
if err != nil {
b.Error(err)
return
}
testMeta.created = int64(num)
offset += n
num, n, err = varint.Unpack64(encodedData[offset:])
if err != nil {
b.Error(err)
return
}
testMeta.modified = int64(num)
offset += n
num, n, err = varint.Unpack64(encodedData[offset:])
if err != nil {
b.Error(err)
return
}
testMeta.expires = int64(num)
offset += n
num, n, err = varint.Unpack64(encodedData[offset:])
if err != nil {
b.Error(err)
return
}
testMeta.deleted = int64(num)
offset += n
switch encodedData[offset] {
case 3:
newMeta.secret = true
newMeta.cronjewel = true
case 2:
newMeta.cronjewel = true
case 1:
newMeta.secret = true
case 0:
default:
b.Errorf("invalid flag value: %d", encodedData[offset])
return
}
}
}
// func BenchmarkMetaSerializeWithXDR2(b *testing.B) {
//
// // Setup
// var w bytes.Buffer
//
// // Reset timer for precise results
// b.ResetTimer()
//
// // Start benchmark
// for i := 0; i < b.N; i++ {
// w.Reset()
// _, err := xdr2.Marshal(&w, testMeta)
// if err != nil {
// b.Errorf("failed to serialize with xdr2: %s", err)
// return
// }
// }
//
// }
// func BenchmarkMetaUnserializeWithXDR2(b *testing.B) {
//
// // Setup
// var w bytes.Buffer
// _, err := xdr2.Marshal(&w, testMeta)
// if err != nil {
// b.Errorf("failed to serialize with xdr2: %s", err)
// }
// encodedData := w.Bytes()
//
// // Reset timer for precise results
// b.ResetTimer()
//
// // Start benchmark
// for i := 0; i < b.N; i++ {
// var newMeta Meta
// _, err := xdr2.Unmarshal(bytes.NewReader(encodedData), &newMeta)
// if err != nil {
// b.Errorf("failed to unserialize with xdr2: %s", err)
// return
// }
// }
//
// }
// func BenchmarkMetaSerializeWithColfer(b *testing.B) {
//
// testColf := &model.Course{
// Created: time.Now().Unix(),
// Modified: time.Now().Unix(),
// Expires: time.Now().Unix(),
// Deleted: time.Now().Unix(),
// Secret: true,
// Cronjewel: true,
// }
//
// // Setup
// for i := 0; i < b.N; i++ {
// _, err := testColf.MarshalBinary()
// if err != nil {
// b.Errorf("failed to serialize with colfer: %s", err)
// return
// }
// }
//
// }
// func BenchmarkMetaUnserializeWithColfer(b *testing.B) {
//
// testColf := &model.Course{
// Created: time.Now().Unix(),
// Modified: time.Now().Unix(),
// Expires: time.Now().Unix(),
// Deleted: time.Now().Unix(),
// Secret: true,
// Cronjewel: true,
// }
// encodedData, err := testColf.MarshalBinary()
// if err != nil {
// b.Errorf("failed to serialize with colfer: %s", err)
// return
// }
//
// // Setup
// for i := 0; i < b.N; i++ {
// var testUnColf model.Course
// err := testUnColf.UnmarshalBinary(encodedData)
// if err != nil {
// b.Errorf("failed to unserialize with colfer: %s", err)
// return
// }
// }
//
// }
func BenchmarkMetaSerializeWithCodegen(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := testMeta.GenCodeMarshal(nil)
if err != nil {
b.Errorf("failed to serialize with codegen: %s", err)
return
}
}
}
func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
// Setup
encodedData, err := testMeta.GenCodeMarshal(nil)
if err != nil {
b.Errorf("failed to serialize with codegen: %s", err)
return
}
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
_, err := newMeta.GenCodeUnmarshal(encodedData)
if err != nil {
b.Errorf("failed to unserialize with codegen: %s", err)
return
}
}
}
func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := dsd.Dump(testMeta, dsd.JSON)
if err != nil {
b.Errorf("failed to serialize with DSD/JSON: %s", err)
return
}
}
}
func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
// Setup
encodedData, err := dsd.Dump(testMeta, dsd.JSON)
if err != nil {
b.Errorf("failed to serialize with DSD/JSON: %s", err)
return
}
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
_, err := dsd.Load(encodedData, &newMeta)
if err != nil {
b.Errorf("failed to unserialize with DSD/JSON: %s", err)
return
}
}
}

157
database/model/meta-gencode.go Normal file
View file

@ -0,0 +1,157 @@
package model
import (
"io"
"time"
"unsafe"
)
var (
_ = unsafe.Sizeof(0)
_ = io.ReadFull
_ = time.Now()
)
// GenCodeSize returns the size of the gencode marshalled byte slice
func (d *Meta) GenCodeSize() (s uint64) {
s += 34
return
}
// GenCodeMarshal gencode marshalls Meta into the given byte array, or a new one if its too small.
func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
size := d.GenCodeSize()
{
if uint64(cap(buf)) >= size {
buf = buf[:size]
} else {
buf = make([]byte, size)
}
}
i := uint64(0)
{
buf[0+0] = byte(d.created >> 0)
buf[1+0] = byte(d.created >> 8)
buf[2+0] = byte(d.created >> 16)
buf[3+0] = byte(d.created >> 24)
buf[4+0] = byte(d.created >> 32)
buf[5+0] = byte(d.created >> 40)
buf[6+0] = byte(d.created >> 48)
buf[7+0] = byte(d.created >> 56)
}
{
buf[0+8] = byte(d.modified >> 0)
buf[1+8] = byte(d.modified >> 8)
buf[2+8] = byte(d.modified >> 16)
buf[3+8] = byte(d.modified >> 24)
buf[4+8] = byte(d.modified >> 32)
buf[5+8] = byte(d.modified >> 40)
buf[6+8] = byte(d.modified >> 48)
buf[7+8] = byte(d.modified >> 56)
}
{
buf[0+16] = byte(d.expires >> 0)
buf[1+16] = byte(d.expires >> 8)
buf[2+16] = byte(d.expires >> 16)
buf[3+16] = byte(d.expires >> 24)
buf[4+16] = byte(d.expires >> 32)
buf[5+16] = byte(d.expires >> 40)
buf[6+16] = byte(d.expires >> 48)
buf[7+16] = byte(d.expires >> 56)
}
{
buf[0+24] = byte(d.deleted >> 0)
buf[1+24] = byte(d.deleted >> 8)
buf[2+24] = byte(d.deleted >> 16)
buf[3+24] = byte(d.deleted >> 24)
buf[4+24] = byte(d.deleted >> 32)
buf[5+24] = byte(d.deleted >> 40)
buf[6+24] = byte(d.deleted >> 48)
buf[7+24] = byte(d.deleted >> 56)
}
{
if d.secret {
buf[32] = 1
} else {
buf[32] = 0
}
}
{
if d.cronjewel {
buf[33] = 1
} else {
buf[33] = 0
}
}
return buf[:i+34], nil
}
// GenCodeUnmarshal gencode unmarshalls Meta and returns the bytes read.
func (d *Meta) GenCodeUnmarshal(buf []byte) (uint64, error) {
i := uint64(0)
{
d.created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
}
{
d.modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
}
{
d.expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
}
{
d.deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
}
{
d.secret = buf[32] == 1
}
{
d.cronjewel = buf[33] == 1
}
return i + 34, nil
}

35
database/model/meta-gencode_test.go Normal file
View file

@ -0,0 +1,35 @@
package model
import (
"reflect"
"testing"
"time"
)
var (
genCodeTestMeta = &Meta{
created: time.Now().Unix(),
modified: time.Now().Unix(),
expires: time.Now().Unix(),
deleted: time.Now().Unix(),
secret: true,
cronjewel: true,
}
)
func TestGenCode(t *testing.T) {
encoded, err := genCodeTestMeta.GenCodeMarshal(nil)
if err != nil {
t.Fatal(err)
}
new := &Meta{}
_, err = new.GenCodeUnmarshal(encoded)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(genCodeTestMeta, new) {
t.Errorf("objects are not equal, got: %v", new)
}
}

163
database/model/meta.gencode.gen.go
View file

@ -1,163 +0,0 @@
package model
import (
"io"
"time"
"unsafe"
)
var (
_ = unsafe.Sizeof(0)
_ = io.ReadFull
_ = time.Now()
)
// type Meta struct {
// Created int64
// Modified int64
// Expires int64
// Deleted int64
// Secret bool
// Cronjewel bool
// }
func (d *Meta) Size() (s uint64) {
s += 34
return
}
func (d *Meta) Marshal(buf []byte) ([]byte, error) {
size := d.Size()
{
if uint64(cap(buf)) >= size {
buf = buf[:size]
} else {
buf = make([]byte, size)
}
}
i := uint64(0)
{
buf[0+0] = byte(d.Created >> 0)
buf[1+0] = byte(d.Created >> 8)
buf[2+0] = byte(d.Created >> 16)
buf[3+0] = byte(d.Created >> 24)
buf[4+0] = byte(d.Created >> 32)
buf[5+0] = byte(d.Created >> 40)
buf[6+0] = byte(d.Created >> 48)
buf[7+0] = byte(d.Created >> 56)
}
{
buf[0+8] = byte(d.Modified >> 0)
buf[1+8] = byte(d.Modified >> 8)
buf[2+8] = byte(d.Modified >> 16)
buf[3+8] = byte(d.Modified >> 24)
buf[4+8] = byte(d.Modified >> 32)
buf[5+8] = byte(d.Modified >> 40)
buf[6+8] = byte(d.Modified >> 48)
buf[7+8] = byte(d.Modified >> 56)
}
{
buf[0+16] = byte(d.Expires >> 0)
buf[1+16] = byte(d.Expires >> 8)
buf[2+16] = byte(d.Expires >> 16)
buf[3+16] = byte(d.Expires >> 24)
buf[4+16] = byte(d.Expires >> 32)
buf[5+16] = byte(d.Expires >> 40)
buf[6+16] = byte(d.Expires >> 48)
buf[7+16] = byte(d.Expires >> 56)
}
{
buf[0+24] = byte(d.Deleted >> 0)
buf[1+24] = byte(d.Deleted >> 8)
buf[2+24] = byte(d.Deleted >> 16)
buf[3+24] = byte(d.Deleted >> 24)
buf[4+24] = byte(d.Deleted >> 32)
buf[5+24] = byte(d.Deleted >> 40)
buf[6+24] = byte(d.Deleted >> 48)
buf[7+24] = byte(d.Deleted >> 56)
}
{
if d.Secret {
buf[32] = 1
} else {
buf[32] = 0
}
}
{
if d.Cronjewel {
buf[33] = 1
} else {
buf[33] = 0
}
}
return buf[:i+34], nil
}
func (d *Meta) Unmarshal(buf []byte) (uint64, error) {
i := uint64(0)
{
d.Created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
}
{
d.Modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
}
{
d.Expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
}
{
d.Deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
}
{
d.Secret = buf[32] == 1
}
{
d.Cronjewel = buf[33] == 1
}
return i + 34, nil
}

74
database/model/meta.go
View file

@ -1,10 +1,72 @@
package model package model
import "time"
// Meta holds
type Meta struct { type Meta struct {
Created int64 `json:"c,omitempty" bson:"c,omitempty"` created int64
Modified int64 `json:"m,omitempty" bson:"m,omitempty"` modified int64
Expires int64 `json:"e,omitempty" bson:"e,omitempty"` expires int64
Deleted int64 `json:"d,omitempty" bson:"d,omitempty"` deleted int64
Secret bool `json:"s,omitempty" bson:"s,omitempty"` // secrets must not be sent to the UI, only synced between nodes secret bool // secrets must not be sent to the UI, only synced between nodes
Cronjewel bool `json:"j,omitempty" bson:"j,omitempty"` // crownjewels must never leave the instance, but may be read by the UI cronjewel bool // crownjewels must never leave the instance, but may be read by the UI
}
// SetAbsoluteExpiry sets an absolute expiry time, that is not affected when the record is updated.
func (m *Meta) SetAbsoluteExpiry(time int64) {
m.expires = time
m.deleted = 0
}
// SetRelativateExpiry sets a relative expiry that is automatically updated whenever the record is updated/saved.
func (m *Meta) SetRelativateExpiry(duration int64) {
if duration >= 0 {
m.deleted = -duration
}
}
// MakeCrownJewel marks the database records as a crownjewel, meaning that it will not be sent/synced to other devices.
func (m *Meta) MakeCrownJewel() {
m.cronjewel = true
}
// MakeSecret sets the database record as secret, meaning that it may only be used internally, and not by interfacing processes, such as the UI.
func (m *Meta) MakeSecret() {
m.secret = true
}
// Update updates the internal meta states and should be called before writing the record to the database.
func (m *Meta) Update() {
now := time.Now().Unix()
m.modified = now
if m.created == 0 {
m.created = now
}
if m.deleted < 0 {
m.expires = now - m.deleted
}
}
// Reset resets all metadata, except for the secret and crownjewel status.
func (m *Meta) Reset() {
m.created = 0
m.modified = 0
m.expires = 0
m.deleted = 0
}
// CheckScope checks whether the current database record exists for the given scope.
func (m *Meta) CheckScope(now int64, local, internal bool) (recordExists bool) {
switch {
case m.deleted > 0:
return false
case m.expires < now:
return false
case !local && m.cronjewel:
return false
case !internal && m.secret:
return false
default:
return true
}
} }

311
database/model/meta_test.go
View file

@ -1,311 +0,0 @@
package model
import (
"bytes"
"testing"
"time"
"github.com/Safing/portbase/container"
"github.com/Safing/portbase/database/model/model"
"github.com/Safing/portbase/formats/dsd"
xdr2 "github.com/davecgh/go-xdr/xdr2"
)
var (
testMeta = &Meta{
Created: time.Now().Unix(),
Modified: time.Now().Unix(),
Expires: time.Now().Unix(),
Deleted: time.Now().Unix(),
Secret: true,
Cronjewel: true,
}
)
func BenchmarkAllocateBytes(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = make([]byte, 33)
}
}
func BenchmarkAllocateStruct1(b *testing.B) {
for i := 0; i < b.N; i++ {
var new Meta
_ = new
}
}
func BenchmarkAllocateStruct2(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = Meta{}
}
}
func BenchmarkMetaSerializeCustom(b *testing.B) {
// Start benchmark
for i := 0; i < b.N; i++ {
c := container.New()
c.AppendNumber(uint64(testMeta.Created))
c.AppendNumber(uint64(testMeta.Modified))
c.AppendNumber(uint64(testMeta.Expires))
c.AppendNumber(uint64(testMeta.Deleted))
switch {
case testMeta.Secret && testMeta.Cronjewel:
c.AppendNumber(3)
case testMeta.Secret:
c.AppendNumber(1)
case testMeta.Cronjewel:
c.AppendNumber(2)
default:
c.AppendNumber(0)
}
}
}
func BenchmarkMetaUnserializeCustom(b *testing.B) {
// Setup
c := container.New()
c.AppendNumber(uint64(testMeta.Created))
c.AppendNumber(uint64(testMeta.Modified))
c.AppendNumber(uint64(testMeta.Expires))
c.AppendNumber(uint64(testMeta.Deleted))
switch {
case testMeta.Secret && testMeta.Cronjewel:
c.AppendNumber(3)
case testMeta.Secret:
c.AppendNumber(1)
case testMeta.Cronjewel:
c.AppendNumber(2)
default:
c.AppendNumber(0)
}
encodedData := c.CompileData()
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
var err error
var num uint64
c := container.New(encodedData)
num, err = c.GetNextN64()
newMeta.Created = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
num, err = c.GetNextN64()
newMeta.Modified = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
num, err = c.GetNextN64()
newMeta.Expires = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
num, err = c.GetNextN64()
newMeta.Deleted = int64(num)
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
flags, err := c.GetNextN8()
if err != nil {
b.Errorf("could not decode: %s", err)
return
}
switch flags {
case 3:
newMeta.Secret = true
newMeta.Cronjewel = true
case 2:
newMeta.Cronjewel = true
case 1:
newMeta.Secret = true
case 0:
default:
b.Errorf("invalid flag value: %d", flags)
return
}
}
}
func BenchmarkMetaSerializeWithXDR2(b *testing.B) {
// Setup
var w bytes.Buffer
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
w.Reset()
_, err := xdr2.Marshal(&w, testMeta)
if err != nil {
b.Errorf("failed to serialize with xdr2: %s", err)
return
}
}
}
func BenchmarkMetaUnserializeWithXDR2(b *testing.B) {
// Setup
var w bytes.Buffer
_, err := xdr2.Marshal(&w, testMeta)
if err != nil {
b.Errorf("failed to serialize with xdr2: %s", err)
}
encodedData := w.Bytes()
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
_, err := xdr2.Unmarshal(bytes.NewReader(encodedData), &newMeta)
if err != nil {
b.Errorf("failed to unserialize with xdr2: %s", err)
return
}
}
}
func BenchmarkMetaSerializeWithColfer(b *testing.B) {
testColf := &model.Course{
Created: time.Now().Unix(),
Modified: time.Now().Unix(),
Expires: time.Now().Unix(),
Deleted: time.Now().Unix(),
Secret: true,
Cronjewel: true,
}
// Setup
for i := 0; i < b.N; i++ {
_, err := testColf.MarshalBinary()
if err != nil {
b.Errorf("failed to serialize with colfer: %s", err)
return
}
}
}
func BenchmarkMetaUnserializeWithColfer(b *testing.B) {
testColf := &model.Course{
Created: time.Now().Unix(),
Modified: time.Now().Unix(),
Expires: time.Now().Unix(),
Deleted: time.Now().Unix(),
Secret: true,
Cronjewel: true,
}
encodedData, err := testColf.MarshalBinary()
if err != nil {
b.Errorf("failed to serialize with colfer: %s", err)
return
}
// Setup
for i := 0; i < b.N; i++ {
var testUnColf model.Course
err := testUnColf.UnmarshalBinary(encodedData)
if err != nil {
b.Errorf("failed to unserialize with colfer: %s", err)
return
}
}
}
func BenchmarkMetaSerializeWithCodegen(b *testing.B) {
for i := 0; i < b.N; i++ {
buf := make([]byte, 34)
_, err := testMeta.Marshal(buf)
if err != nil {
b.Errorf("failed to serialize with codegen: %s", err)
return
}
}
}
func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
// Setup
buf := make([]byte, 34)
encodedData, err := testMeta.Marshal(buf)
if err != nil {
b.Errorf("failed to serialize with codegen: %s", err)
return
}
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
_, err := newMeta.Unmarshal(encodedData)
if err != nil {
b.Errorf("failed to unserialize with codegen: %s", err)
return
}
}
}
func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := dsd.Dump(testMeta, dsd.JSON)
if err != nil {
b.Errorf("failed to serialize with DSD/JSON: %s", err)
return
}
}
}
func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
// Setup
encodedData, err := dsd.Dump(testMeta, dsd.JSON)
if err != nil {
b.Errorf("failed to serialize with DSD/JSON: %s", err)
return
}
// Reset timer for precise results
b.ResetTimer()
// Start benchmark
for i := 0; i < b.N; i++ {
var newMeta Meta
_, err := dsd.Load(encodedData, &newMeta)
if err != nil {
b.Errorf("failed to unserialize with DSD/JSON: %s", err)
return
}
}
}

7
database/model/model.go
View file

@ -1,4 +1,11 @@
package model package model
// Model provides an interface for uniformally handling database records.
type Model interface { type Model interface {
Key() string
SetKey(key string)
MoveTo(key string)
Meta() *Meta
SetMeta(meta *Meta)
Marshal(format uint8) ([]byte, error)
} }

67
database/model/wrapper.go Normal file
View file

@ -0,0 +1,67 @@
package model
import (
"errors"
"fmt"
"github.com/Safing/safing-core/formats/dsd"
"github.com/Safing/safing-core/formats/varint"
)
type Wrapper struct {
dbName string
dbKey string
meta *Meta
Format uint8
Data []byte
}
func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
format, _, err := varint.Unpack8(data)
if err != nil {
return nil, fmt.Errorf("database: could not get dsd format: %s", err)
}
new := &Wrapper{
dbKey: key,
meta: meta,
Format: format,
Data: data,
}
return new, nil
}
// Key returns the key of the database record.
func (w *Wrapper) Key() string {
return w.dbKey
}
// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
func (w *Wrapper) SetKey(key string) {
w.dbKey = key
}
// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
func (w *Wrapper) MoveTo(key string) {
w.dbKey = key
w.meta.Reset()
}
// Meta returns the metadata object for this record.
func (w *Wrapper) Meta() *Meta {
return w.meta
}
// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
func (w *Wrapper) SetMeta(meta *Meta) {
w.meta = meta
}
// Marshal marshals the object, without the database key or metadata
func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
if storageType != dsd.AUTO && storageType != w.Format {
return nil, errors.New("could not dump model, wrapped object format mismatch")
}
return w.Data, nil
}

42
database/storage/interface.go
View file

@ -1,30 +1,34 @@
package storage package storage
import ( import (
"github.com/Safing/portbase/database/iterator" "github.com/Safing/portbase/database/iterator"
"github.com/Safing/portbase/database/model" "github.com/Safing/portbase/database/model"
"github.com/Safing/portbase/database/query"
) )
// Interface defines the database storage API. // Interface defines the database storage API.
type Interface interface { type Interface interface {
// Full // Retrieve
Exists(key string) (bool, error) Exists(key string) (bool, error)
Get(key string) (model.Model, error) Get(key string) (model.Model, error)
Create(key string, model model.Model) error
Update(key string, model model.Model) error // create when not exists
UpdateOrCreate(key string, model model.Model) error // update, create if not exists.
Delete(key string) error
// Partial // Modify
// What happens if I mutate a value that does not yet exist? How would I know its type? Create(model model.Model) error
InsertPartial(key string, partialObject interface{}) Update(model model.Model) error // create when not exists
InsertValue(key string, attribute string, value interface{}) UpdateOrCreate(model model.Model) error // update, create if not exists.
Delete(key string) error
// Query // Partial
Query(*query.Query) (*iterator.Iterator, error) // What happens if I mutate a value that does not yet exist? How would I know its type?
InsertPartial(key string, partialObject interface{})
InsertValue(key string, attribute string, value interface{})
// Meta // Query
LetExpire(key string, timestamp int64) error Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
MakeSecret(key string) error // only visible internal
MakeCrownJewel(key string) error // do not sync between devices // Meta
SetAbsoluteExpiry(key string, time int64)
SetRelativateExpiry(key string, duration int64)
MakeCrownJewel(key string)
MakeSecret(key string)
} }

36
database/storages.go Normal file
View file

@ -0,0 +1,36 @@
package database
// A Factory creates a new database of it's type.
type Factory func(name, location string) (*storage.Interface, error)
var (
storages map[string]Factory
storagesLock sync.Mutex
)
// RegisterStorage registers a new storage type.
func RegisterStorage(name string, factory Factory) error {
storagesLock.Lock()
defer storagesLock.Unlock()
_, ok := storages[name]
if ok {
return errors.New("factory for this type already exists")
}
storages[name] = factory
return nil
}
// startDatabase starts a new database with the given name, storageType at location.
func startDatabase(name, storageType, location string) (*storage.Interface, error) {
storagesLock.Lock()
defer storagesLock.Unlock()
factory, ok := storages[name]
if !ok {
return fmt.Errorf("storage of this type (%s) does not exist", storageType)
}
return factory(name, location)
}