Finalize model

database/controller.go

@@ -0,0 +1,38 @@
+package database
+
+type Controller struct {
+  storage
+  writeLock sync.RWMutex
+  readLock sync.RWMutex
+  migrating *abool.AtomicBool
+}
+
+func NewController() (*Controller, error) {
+
+}
+
+	// Retrieve
+func (c *Controller) Exists(key string) (bool, error) {}
+func (c *Controller) Get(key string) (model.Model, error) {}
+
+// Modify
+func (c *Controller) Create(model model.Model) error {}
+// create when not exists
+func (c *Controller) Update(model model.Model) error {}
+// update, create if not exists.
+func (c *Controller) UpdateOrCreate(model model.Model) error {}
+func (c *Controller) Delete(key string) error {}
+
+// Partial
+// What happens if I mutate a value that does not yet exist? How would I know its type?
+func (c *Controller) InsertPartial(key string, partialObject interface{}) {}
+func (c *Controller) InsertValue(key string, attribute string, value interface{}) {}
+
+// Query
+func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {}
+
+// Meta
+func (c *Controller) SetAbsoluteExpiry(key string, time int64) {}
+func (c *Controller) SetRelativateExpiry(key string, duration int64) {}
+func (c *Controller) MakeCrownJewel(key string) {}
+func (c *Controller) MakeSecret(key string) {}

database/databases.go

@@ -2,11 +2,11 @@ package database
 
 
 var (
-  databases = make(map[string]*storage.Interface)
+  databases = make(map[string]*Controller)
   databasesLock sync.Mutex
 )
 
-func getDatabase(name string) *storage.Interface {
+func getDatabase(name string) *Controller {
   databasesLock.Lock()
   defer   databasesLock.Unlock()
   storage, ok := databases[name]

database/interface.go

@@ -1,17 +1,27 @@
 package database
 
-type Interface struct {
+// Interface provides a method to access the database with attached options.
-  local bool
+type Interface struct {}
-  internal bool
+
+// Options holds options that may be set for an Interface instance.
+type Options struct {
+  Local bool
+  Internal bool
+  AlwaysMakeSecret bool
+  AlwaysMakeCrownjewel bool
 }
 
-func NewInterface(local bool, internal bool) *Interface {
+// NewInterface returns a new Interface to the database.
+func NewInterface(opts *Options) *Interface {
   return &Interface{
     local: local,
     internal: internal,
   }
 }
 
-func (i *Interface) Get(string key) (model.Model, error) {
+func (i *Interface) Get(key string) (model.Model, error) {
+
+  controller 
+
   return nil, nil
 }

database/model/base.go

@@ -1,6 +1,10 @@
 package model
 
 import (
+	"errors"
+	"fmt"
+
+	"github.com/Safing/portbase/container"
 	"github.com/Safing/portbase/formats/dsd"
 )
 
@@ -13,17 +17,27 @@ type Base struct {
 
 // Key returns the key of the database record.
 func (b *Base) Key() string {
+	return fmt.Sprintf("%s:%s", b.dbName, b.dbKey)
+}
+
+// DatabaseName returns the name of the database.
+func (b *Base) DatabaseName() string {
+	return b.dbName
+}
+
+// DatabaseKey returns the database key of the database record.
+func (b *Base) DatabaseKey() string {
 	return b.dbKey
 }
 
 // SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
 func (b *Base) SetKey(key string) {
-	b.dbKey = key
+	b.dbName, b.dbKey = ParseKey(key)
 }
 
 // MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
 func (b *Base) MoveTo(key string) {
-	b.dbKey = key
+	b.SetKey(key)
 	b.meta.Reset()
 }
 
@@ -45,3 +59,29 @@ func (b *Base) Marshal(format uint8) ([]byte, error) {
 	}
 	return dumped, nil
 }
+
+// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+func (b *Base) MarshalRecord() ([]byte, error) {
+	if b.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	// version
+	c := container.New([]byte{1})
+
+	// meta
+	metaSection, err := b.meta.GenCodeMarshal(nil)
+	if err != nil {
+		return nil, err
+	}
+	c.AppendAsBlock(metaSection)
+
+	// data
+	dataSection, err := b.Marshal(dsd.JSON)
+	if err != nil {
+		return nil, err
+	}
+	c.Append(dataSection)
+
+	return c.CompileData(), nil
+}

database/model/base_test.go

@@ -0,0 +1,13 @@
+package model
+
+import "testing"
+
+func TestBaseModel(t *testing.T) {
+
+	// check model interface compliance
+	var m Model
+	b := &TestModel{}
+	m = b
+	_ = m
+
+}

database/model/key.go

@@ -1,14 +1,14 @@
 package model
 
 import (
-	"errors"
 	"strings"
 )
 
-func ParseKey(key string) (dbName, dbKey string, err error) {
+// ParseKey splits a key into it's database name and key parts.
+func ParseKey(key string) (dbName, dbKey string) {
 	splitted := strings.SplitN(key, ":", 2)
 	if len(splitted) == 2 {
-		return splitted[0], splitted[1], nil
+		return splitted[0], splitted[1]
 	}
-	return "", "", errors.New("invalid key")
+	return splitted[0], ""
 }

database/model/meta-gencode.go

@@ -1,6 +1,7 @@
 package model
 
 import (
+	"fmt"
 	"io"
 	"time"
 	"unsafe"
@@ -13,7 +14,7 @@ var (
 )
 
 // GenCodeSize returns the size of the gencode marshalled byte slice
-func (d *Meta) GenCodeSize() (s uint64) {
+func (d *Meta) GenCodeSize() (s int) {
 	s += 34
 	return
 }
@@ -22,7 +23,7 @@ func (d *Meta) GenCodeSize() (s uint64) {
 func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
 	size := d.GenCodeSize()
 	{
-		if uint64(cap(buf)) >= size {
+		if cap(buf) >= size {
 			buf = buf[:size]
 		} else {
 			buf = make([]byte, size)
@@ -125,6 +126,10 @@ func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
 
 // GenCodeUnmarshal gencode unmarshalls Meta and returns the bytes read.
 func (d *Meta) GenCodeUnmarshal(buf []byte) (uint64, error) {
+	if len(buf) < d.GenCodeSize() {
+		return 0, fmt.Errorf("insufficient data: got %d out of %d bytes", len(buf), d.GenCodeSize())
+	}
+
 	i := uint64(0)
 
 	{

database/model/meta.go

@@ -12,19 +12,37 @@ type Meta struct {
 	cronjewel bool // crownjewels must never leave the instance, but may be read by the UI
 }
 
-// SetAbsoluteExpiry sets an absolute expiry time, that is not affected when the record is updated.
+// SetAbsoluteExpiry sets an absolute expiry time (in seconds), that is not affected when the record is updated.
-func (m *Meta) SetAbsoluteExpiry(time int64) {
+func (m *Meta) SetAbsoluteExpiry(seconds int64) {
-	m.expires = time
+	m.expires = seconds
 	m.deleted = 0
 }
 
-// SetRelativateExpiry sets a relative expiry that is automatically updated whenever the record is updated/saved.
+// SetRelativateExpiry sets a relative expiry time (ie. TTL in seconds) that is automatically updated whenever the record is updated/saved.
-func (m *Meta) SetRelativateExpiry(duration int64) {
+func (m *Meta) SetRelativateExpiry(seconds int64) {
-	if duration >= 0 {
+	if seconds >= 0 {
-		m.deleted = -duration
+		m.deleted = -seconds
 	}
 }
 
+// GetAbsoluteExpiry returns the absolute expiry time.
+func (m *Meta) GetAbsoluteExpiry() int64 {
+	return m.expires
+}
+
+// GetRelativeExpiry returns the current relative expiry time - ie. seconds until expiry.
+func (m *Meta) GetRelativeExpiry() int64 {
+	if m.deleted < 0 {
+		return -m.deleted
+	}
+
+	abs := m.expires - time.Now().Unix()
+	if abs < 0 {
+		return 0
+	}
+	return abs
+}
+
 // MakeCrownJewel marks the database records as a crownjewel, meaning that it will not be sent/synced to other devices.
 func (m *Meta) MakeCrownJewel() {
 	m.cronjewel = true

database/model/model.go

@@ -2,10 +2,18 @@ package model
 
 // Model provides an interface for uniformally handling database records.
 type Model interface {
-	Key() string
+	Key() string          // test:config
-	SetKey(key string)
+	DatabaseName() string // test
-	MoveTo(key string)
+	DatabaseKey() string  // config
+
+	SetKey(key string) // test:config
+	MoveTo(key string) // test:config
 	Meta() *Meta
 	SetMeta(meta *Meta)
+
 	Marshal(format uint8) ([]byte, error)
+	MarshalRecord() ([]byte, error)
+
+	Lock()
+	Unlock()
 }

database/model/model_test.go

@@ -0,0 +1,16 @@
+package model
+
+import "sync"
+
+type TestModel struct {
+	Base
+	lock sync.Mutex
+}
+
+func (tm *TestModel) Lock() {
+	tm.lock.Lock()
+}
+
+func (tm *TestModel) Unlock() {
+	tm.lock.Unlock()
+}

database/model/wrapper.go

@@ -3,59 +3,74 @@ package model
 import (
 	"errors"
 	"fmt"
+	"sync"
 
-	"github.com/Safing/safing-core/formats/dsd"
+	"github.com/Safing/portbase/container"
-	"github.com/Safing/safing-core/formats/varint"
+	"github.com/Safing/portbase/formats/dsd"
+	"github.com/Safing/portbase/formats/varint"
 )
 
 type Wrapper struct {
-	dbName string
+	Base
-	dbKey  string
-	meta   *Meta
 	Format uint8
 	Data   []byte
+	lock   sync.Mutex
 }
 
+func NewRawWrapper(database, key string, data []byte) (*Wrapper, error) {
+	version, offset, err := varint.Unpack8(data)
+	if version != 1 {
+		return nil, fmt.Errorf("incompatible record version: %d", version)
+	}
+
+	metaSection, n, err := varint.GetNextBlock(data[offset:])
+	if err != nil {
+		return nil, fmt.Errorf("could not get meta section: %s", err)
+	}
+	offset += n
+
+	newMeta := &Meta{}
+	_, err = newMeta.GenCodeUnmarshal(metaSection)
+	if err != nil {
+		return nil, fmt.Errorf("could not unmarshal meta section: %s", err)
+	}
+
+	format, _, err := varint.Unpack8(data[offset:])
+	if err != nil {
+		return nil, fmt.Errorf("could not get dsd format: %s", err)
+	}
+
+	return &Wrapper{
+		Base{
+			database,
+			key,
+			newMeta,
+		},
+		format,
+		data[offset:],
+		sync.Mutex{},
+	}, nil
+}
+
+// NewWrapper returns a new model wrapper for the given data.
 func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
 	format, _, err := varint.Unpack8(data)
 	if err != nil {
-		return nil, fmt.Errorf("database: could not get dsd format: %s", err)
+		return nil, fmt.Errorf("could not get dsd format: %s", err)
 	}
 
-	new := &Wrapper{
+	dbName, dbKey := ParseKey(key)
-		dbKey:  key,
-		meta:   meta,
-		Format: format,
-		Data:   data,
-	}
 
-	return new, nil
+	return &Wrapper{
-}
+		Base{
-
+			dbName: dbName,
-// Key returns the key of the database record.
+			dbKey:  dbKey,
-func (w *Wrapper) Key() string {
+			meta:   meta,
-	return w.dbKey
+		},
-}
+		format,
-
+		data,
-// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
+		sync.Mutex{},
-func (w *Wrapper) SetKey(key string) {
+	}, nil
-	w.dbKey = key
-}
-
-// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
-func (w *Wrapper) MoveTo(key string) {
-	w.dbKey = key
-	w.meta.Reset()
-}
-
-// Meta returns the metadata object for this record.
-func (w *Wrapper) Meta() *Meta {
-	return w.meta
-}
-
-// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
-func (w *Wrapper) SetMeta(meta *Meta) {
-	w.meta = meta
 }
 
 // Marshal marshals the object, without the database key or metadata
@@ -65,3 +80,41 @@ func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
 	}
 	return w.Data, nil
 }
+
+// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+func (w *Wrapper) MarshalRecord() ([]byte, error) {
+	// Duplication necessary, as the version from Base would call Base.Marshal instead of Wrapper.Marshal
+
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	// version
+	c := container.New([]byte{1})
+
+	// meta
+	metaSection, err := w.meta.GenCodeMarshal(nil)
+	if err != nil {
+		return nil, err
+	}
+	c.AppendAsBlock(metaSection)
+
+	// data
+	dataSection, err := w.Marshal(dsd.JSON)
+	if err != nil {
+		return nil, err
+	}
+	c.Append(dataSection)
+
+	return c.CompileData(), nil
+}
+
+// Lock locks the record.
+func (w *Wrapper) Lock() {
+	w.lock.Lock()
+}
+
+// Unlock unlocks the record.
+func (w *Wrapper) Unlock() {
+	w.lock.Unlock()
+}

database/model/wrapper_test.go

@@ -0,0 +1,55 @@
+package model
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/Safing/portbase/formats/dsd"
+)
+
+func TestWrapper(t *testing.T) {
+
+	// check model interface compliance
+	var m Model
+	w := &Wrapper{}
+	m = w
+	_ = m
+
+	// create test data
+	testData := []byte(`J{"a": "b"}`)
+
+	// test wrapper
+	wrapper, err := NewWrapper("test:a", nil, testData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if wrapper.Format != dsd.JSON {
+		t.Error("format mismatch")
+	}
+	if !bytes.Equal(testData, wrapper.Data) {
+		t.Error("data mismatch")
+	}
+
+	encoded, err := wrapper.Marshal(dsd.JSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(testData, encoded) {
+		t.Error("marshal mismatch")
+	}
+
+	wrapper.SetMeta(&Meta{})
+	raw, err := wrapper.MarshalRecord()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wrapper2, err := NewRawWrapper("test", "a", raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(testData, wrapper2.Data) {
+		t.Error("marshal mismatch")
+	}
+
+}

database/storages.go

@@ -1,36 +0,0 @@
-package database
-
-// A Factory creates a new database of it's type.
-type Factory func(name, location string) (*storage.Interface, error)
-
-var (
-  storages map[string]Factory
-  storagesLock sync.Mutex
-)
-
-// RegisterStorage registers a new storage type.
-func RegisterStorage(name string, factory Factory) error {
-  storagesLock.Lock()
-  defer storagesLock.Unlock()
-
-  _, ok := storages[name]
-  if ok {
-    return errors.New("factory for this type already exists")
-  }
-
-  storages[name] = factory
-  return nil
-}
-
-// startDatabase starts a new database with the given name, storageType at location.
-func startDatabase(name, storageType, location string) (*storage.Interface, error) {
-  storagesLock.Lock()
-  defer storagesLock.Unlock()
-
-  factory, ok := storages[name]
-  if !ok {
-    return fmt.Errorf("storage of this type (%s) does not exist", storageType)
-  }
-
-  return factory(name, location)
-}