From 6ed50f34fbf7f5dcdcbd5aa83b898debaea3fc5a Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 29 Aug 2018 20:02:02 +0200
Subject: [PATCH 01/20] Add query package with first set of conditions and
 tests

---
 database/query/condition-and.go         |  31 +++++++
 database/query/condition-bool.go        |  64 +++++++++++++++
 database/query/condition-error.go       |  19 +++++
 database/query/condition-exists.go      |  28 +++++++
 database/query/condition-float.go       |  90 ++++++++++++++++++++
 database/query/condition-int.go         |  86 ++++++++++++++++++++
 database/query/condition-not.go         |  20 +++++
 database/query/condition-or.go          |  31 +++++++
 database/query/condition-regex.go       |  57 +++++++++++++
 database/query/condition-string.go      |  56 +++++++++++++
 database/query/condition-stringslice.go |  60 ++++++++++++++
 database/query/condition.go             |  67 +++++++++++++++
 database/query/fetcher.go               |  78 ++++++++++++++++++
 database/query/parser.go                |  44 ++++++++++
 database/query/query.go                 |  98 ++++++++++++++++++++++
 database/query/query_test.go            | 104 ++++++++++++++++++++++++
 16 files changed, 933 insertions(+)
 create mode 100644 database/query/condition-and.go
 create mode 100644 database/query/condition-bool.go
 create mode 100644 database/query/condition-error.go
 create mode 100644 database/query/condition-exists.go
 create mode 100644 database/query/condition-float.go
 create mode 100644 database/query/condition-int.go
 create mode 100644 database/query/condition-not.go
 create mode 100644 database/query/condition-or.go
 create mode 100644 database/query/condition-regex.go
 create mode 100644 database/query/condition-string.go
 create mode 100644 database/query/condition-stringslice.go
 create mode 100644 database/query/condition.go
 create mode 100644 database/query/fetcher.go
 create mode 100644 database/query/parser.go
 create mode 100644 database/query/query.go
 create mode 100644 database/query/query_test.go

diff --git a/database/query/condition-and.go b/database/query/condition-and.go
new file mode 100644
index 0000000..913cbcf
--- /dev/null
+++ b/database/query/condition-and.go
@@ -0,0 +1,31 @@
+package query
+
+// And combines multiple conditions with a logical _AND_ operator.
+func And(conditions ...Condition) Condition {
+	return &andCond{
+		conditions: conditions,
+	}
+}
+
+type andCond struct {
+	conditions []Condition
+}
+
+func (c *andCond) complies(f Fetcher) bool {
+	for _, cond := range c.conditions {
+		if !cond.complies(f) {
+			return false
+		}
+	}
+	return true
+}
+
+func (c *andCond) check() (err error) {
+	for _, cond := range c.conditions {
+		err = cond.check()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/database/query/condition-bool.go b/database/query/condition-bool.go
new file mode 100644
index 0000000..2e6bec8
--- /dev/null
+++ b/database/query/condition-bool.go
@@ -0,0 +1,64 @@
+package query
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+)
+
+type boolCondition struct {
+	key      string
+	operator uint8
+	value    bool
+}
+
+func newBoolCondition(key string, operator uint8, value interface{}) *boolCondition {
+
+	var parsedValue bool
+
+	switch v := value.(type) {
+	case bool:
+		parsedValue = v
+	case string:
+		var err error
+		parsedValue, err = strconv.ParseBool(v)
+		if err != nil {
+			return &boolCondition{
+				key:      fmt.Sprintf("could not parse \"%s\" to bool: %s", v, err),
+				operator: errorPresent,
+			}
+		}
+	default:
+		return &boolCondition{
+			key:      fmt.Sprintf("incompatible value %v for int64", value),
+			operator: errorPresent,
+		}
+	}
+
+	return &boolCondition{
+		key:      key,
+		operator: operator,
+		value:    parsedValue,
+	}
+}
+
+func (c *boolCondition) complies(f Fetcher) bool {
+	comp, ok := f.GetBool(c.key)
+	if !ok {
+		return false
+	}
+
+	switch c.operator {
+	case Is:
+		return comp == c.value
+	default:
+		return false
+	}
+}
+
+func (c *boolCondition) check() error {
+	if c.operator == errorPresent {
+		return errors.New(c.key)
+	}
+	return nil
+}
diff --git a/database/query/condition-error.go b/database/query/condition-error.go
new file mode 100644
index 0000000..a8c7610
--- /dev/null
+++ b/database/query/condition-error.go
@@ -0,0 +1,19 @@
+package query
+
+type errorCondition struct {
+	err error
+}
+
+func newErrorCondition(err error) *errorCondition {
+	return &errorCondition{
+		err: err,
+	}
+}
+
+func (c *errorCondition) complies(f Fetcher) bool {
+	return false
+}
+
+func (c *errorCondition) check() error {
+	return c.err
+}
diff --git a/database/query/condition-exists.go b/database/query/condition-exists.go
new file mode 100644
index 0000000..086f200
--- /dev/null
+++ b/database/query/condition-exists.go
@@ -0,0 +1,28 @@
+package query
+
+import (
+	"errors"
+)
+
+type existsCondition struct {
+	key      string
+	operator uint8
+}
+
+func newExistsCondition(key string, operator uint8) *existsCondition {
+	return &existsCondition{
+		key:      key,
+		operator: operator,
+	}
+}
+
+func (c *existsCondition) complies(f Fetcher) bool {
+	return f.Exists(c.key)
+}
+
+func (c *existsCondition) check() error {
+	if c.operator == errorPresent {
+		return errors.New(c.key)
+	}
+	return nil
+}
diff --git a/database/query/condition-float.go b/database/query/condition-float.go
new file mode 100644
index 0000000..3627b91
--- /dev/null
+++ b/database/query/condition-float.go
@@ -0,0 +1,90 @@
+package query
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+)
+
+type floatCondition struct {
+	key      string
+	operator uint8
+	value    float64
+}
+
+func newFloatCondition(key string, operator uint8, value interface{}) *floatCondition {
+
+	var parsedValue float64
+
+	switch v := value.(type) {
+	case int:
+		parsedValue = float64(v)
+	case int8:
+		parsedValue = float64(v)
+	case int16:
+		parsedValue = float64(v)
+	case int32:
+		parsedValue = float64(v)
+	case int64:
+		parsedValue = float64(v)
+	case uint:
+		parsedValue = float64(v)
+	case uint8:
+		parsedValue = float64(v)
+	case uint32:
+		parsedValue = float64(v)
+	case float32:
+		parsedValue = float64(v)
+	case float64:
+		parsedValue = v
+	case string:
+		var err error
+		parsedValue, err = strconv.ParseFloat(v, 64)
+		if err != nil {
+			return &floatCondition{
+				key:      fmt.Sprintf("could not parse %s to float64: %s", v, err),
+				operator: errorPresent,
+			}
+		}
+	default:
+		return &floatCondition{
+			key:      fmt.Sprintf("incompatible value %v for float64", value),
+			operator: errorPresent,
+		}
+	}
+
+	return &floatCondition{
+		key:      key,
+		operator: operator,
+		value:    parsedValue,
+	}
+}
+
+func (c *floatCondition) complies(f Fetcher) bool {
+	comp, ok := f.GetFloat(c.key)
+	if !ok {
+		return false
+	}
+
+	switch c.operator {
+	case FloatEquals:
+		return comp == c.value
+	case FloatGreaterThan:
+		return comp > c.value
+	case FloatGreaterThanOrEqual:
+		return comp >= c.value
+	case FloatLessThan:
+		return comp < c.value
+	case FloatLessThanOrEqual:
+		return comp <= c.value
+	default:
+		return false
+	}
+}
+
+func (c *floatCondition) check() error {
+	if c.operator == errorPresent {
+		return errors.New(c.key)
+	}
+	return nil
+}
diff --git a/database/query/condition-int.go b/database/query/condition-int.go
new file mode 100644
index 0000000..b169a12
--- /dev/null
+++ b/database/query/condition-int.go
@@ -0,0 +1,86 @@
+package query
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+)
+
+type intCondition struct {
+	key      string
+	operator uint8
+	value    int64
+}
+
+func newIntCondition(key string, operator uint8, value interface{}) *intCondition {
+
+	var parsedValue int64
+
+	switch v := value.(type) {
+	case int:
+		parsedValue = int64(v)
+	case int8:
+		parsedValue = int64(v)
+	case int16:
+		parsedValue = int64(v)
+	case int32:
+		parsedValue = int64(v)
+	case int64:
+		parsedValue = int64(v)
+	case uint:
+		parsedValue = int64(v)
+	case uint8:
+		parsedValue = int64(v)
+	case uint32:
+		parsedValue = int64(v)
+	case string:
+		var err error
+		parsedValue, err = strconv.ParseInt(v, 10, 64)
+		if err != nil {
+			return &intCondition{
+				key:      fmt.Sprintf("could not parse %s to int64: %s (hint: use \"sameas\" to compare strings)", v, err),
+				operator: errorPresent,
+			}
+		}
+	default:
+		return &intCondition{
+			key:      fmt.Sprintf("incompatible value %v for int64", value),
+			operator: errorPresent,
+		}
+	}
+
+	return &intCondition{
+		key:      key,
+		operator: operator,
+		value:    parsedValue,
+	}
+}
+
+func (c *intCondition) complies(f Fetcher) bool {
+	comp, ok := f.GetInt(c.key)
+	if !ok {
+		return false
+	}
+
+	switch c.operator {
+	case Equals:
+		return comp == c.value
+	case GreaterThan:
+		return comp > c.value
+	case GreaterThanOrEqual:
+		return comp >= c.value
+	case LessThan:
+		return comp < c.value
+	case LessThanOrEqual:
+		return comp <= c.value
+	default:
+		return false
+	}
+}
+
+func (c *intCondition) check() error {
+	if c.operator == errorPresent {
+		return errors.New(c.key)
+	}
+	return nil
+}
diff --git a/database/query/condition-not.go b/database/query/condition-not.go
new file mode 100644
index 0000000..7249568
--- /dev/null
+++ b/database/query/condition-not.go
@@ -0,0 +1,20 @@
+package query
+
+// Not negates the supplied condition.
+func Not(c Condition) Condition {
+	return &notCond{
+		notC: c,
+	}
+}
+
+type notCond struct {
+	notC Condition
+}
+
+func (c *notCond) complies(f Fetcher) bool {
+	return !c.notC.complies(f)
+}
+
+func (c *notCond) check() error {
+	return c.notC.check()
+}
diff --git a/database/query/condition-or.go b/database/query/condition-or.go
new file mode 100644
index 0000000..e93228b
--- /dev/null
+++ b/database/query/condition-or.go
@@ -0,0 +1,31 @@
+package query
+
+// Or combines multiple conditions with a logical _OR_ operator.
+func Or(conditions ...Condition) Condition {
+	return &orCond{
+		conditions: conditions,
+	}
+}
+
+type orCond struct {
+	conditions []Condition
+}
+
+func (c *orCond) complies(f Fetcher) bool {
+	for _, cond := range c.conditions {
+		if cond.complies(f) {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *orCond) check() (err error) {
+	for _, cond := range c.conditions {
+		err = cond.check()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/database/query/condition-regex.go b/database/query/condition-regex.go
new file mode 100644
index 0000000..b32d212
--- /dev/null
+++ b/database/query/condition-regex.go
@@ -0,0 +1,57 @@
+package query
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+)
+
+type regexCondition struct {
+	key      string
+	operator uint8
+	regex    *regexp.Regexp
+}
+
+func newRegexCondition(key string, operator uint8, value interface{}) *regexCondition {
+	switch v := value.(type) {
+	case string:
+		r, err := regexp.Compile(v)
+		if err != nil {
+			return &regexCondition{
+				key:      fmt.Sprintf("could not compile regex \"%s\": %s", v, err),
+				operator: errorPresent,
+			}
+		}
+		return &regexCondition{
+			key:      key,
+			operator: operator,
+			regex:    r,
+		}
+	default:
+		return &regexCondition{
+			key:      fmt.Sprintf("incompatible value %v for string", value),
+			operator: errorPresent,
+		}
+	}
+}
+
+func (c *regexCondition) complies(f Fetcher) bool {
+	comp, ok := f.GetString(c.key)
+	if !ok {
+		return false
+	}
+
+	switch c.operator {
+	case Matches:
+		return c.regex.MatchString(comp)
+	default:
+		return false
+	}
+}
+
+func (c *regexCondition) check() error {
+	if c.operator == errorPresent {
+		return errors.New(c.key)
+	}
+	return nil
+}
diff --git a/database/query/condition-string.go b/database/query/condition-string.go
new file mode 100644
index 0000000..f255693
--- /dev/null
+++ b/database/query/condition-string.go
@@ -0,0 +1,56 @@
+package query
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+type stringCondition struct {
+	key      string
+	operator uint8
+	value    string
+}
+
+func newStringCondition(key string, operator uint8, value interface{}) *stringCondition {
+	switch v := value.(type) {
+	case string:
+		return &stringCondition{
+			key:      key,
+			operator: operator,
+			value:    v,
+		}
+	default:
+		return &stringCondition{
+			key:      fmt.Sprintf("incompatible value %v for string", value),
+			operator: errorPresent,
+		}
+	}
+}
+
+func (c *stringCondition) complies(f Fetcher) bool {
+	comp, ok := f.GetString(c.key)
+	if !ok {
+		return false
+	}
+
+	switch c.operator {
+	case Matches:
+		return c.value == comp
+	case Contains:
+		return strings.Contains(comp, c.value)
+	case StartsWith:
+		return strings.HasPrefix(comp, c.value)
+	case EndsWith:
+		return strings.HasSuffix(comp, c.value)
+	default:
+		return false
+	}
+}
+
+func (c *stringCondition) check() error {
+	if c.operator == errorPresent {
+		return errors.New(c.key)
+	}
+	return nil
+}
diff --git a/database/query/condition-stringslice.go b/database/query/condition-stringslice.go
new file mode 100644
index 0000000..cfd04d4
--- /dev/null
+++ b/database/query/condition-stringslice.go
@@ -0,0 +1,60 @@
+package query
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/Safing/portbase/utils"
+)
+
+type stringSliceCondition struct {
+	key      string
+	operator uint8
+	value    []string
+}
+
+func newStringSliceCondition(key string, operator uint8, value interface{}) *stringSliceCondition {
+
+	switch v := value.(type) {
+	case string:
+		parsedValue := strings.Split(v, ",")
+		if len(parsedValue) < 2 {
+			return &stringSliceCondition{
+				key:      fmt.Sprintf("could not parse \"%s\" to []string", v),
+				operator: errorPresent,
+			}
+		}
+		return &stringSliceCondition{
+			key:      key,
+			operator: operator,
+			value:    parsedValue,
+		}
+	default:
+		return &stringSliceCondition{
+			key:      fmt.Sprintf("incompatible value %v for []string", value),
+			operator: errorPresent,
+		}
+	}
+
+}
+
+func (c *stringSliceCondition) complies(f Fetcher) bool {
+	comp, ok := f.GetString(c.key)
+	if !ok {
+		return false
+	}
+
+	switch c.operator {
+	case In:
+		return utils.StringInSlice(c.value, comp)
+	default:
+		return false
+	}
+}
+
+func (c *stringSliceCondition) check() error {
+	if c.operator == errorPresent {
+		return fmt.Errorf("could not parse \"%s\" to []string", c.key)
+	}
+	return nil
+}
diff --git a/database/query/condition.go b/database/query/condition.go
new file mode 100644
index 0000000..85ae28b
--- /dev/null
+++ b/database/query/condition.go
@@ -0,0 +1,67 @@
+package query
+
+import "fmt"
+
+// Condition is an interface to provide a common api to all condition types.
+type Condition interface {
+	complies(f Fetcher) bool
+	check() error
+	// string() string
+}
+
+// Operators
+const (
+	Equals                  uint8 = iota // int
+	GreaterThan                          // int
+	GreaterThanOrEqual                   // int
+	LessThan                             // int
+	LessThanOrEqual                      // int
+	FloatEquals                          // float
+	FloatGreaterThan                     // float
+	FloatGreaterThanOrEqual              // float
+	FloatLessThan                        // float
+	FloatLessThanOrEqual                 // float
+	SameAs                               // string
+	Contains                             // string
+	StartsWith                           // string
+	EndsWith                             // string
+	In                                   // stringSlice
+	Matches                              // regex
+	Is                                   // bool: accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE
+	Exists                               // any
+
+	errorPresent uint8 = 255
+)
+
+// Where returns a condition to add to a query.
+func Where(key string, operator uint8, value interface{}) Condition {
+	switch operator {
+	case Equals,
+		GreaterThan,
+		GreaterThanOrEqual,
+		LessThan,
+		LessThanOrEqual:
+		return newIntCondition(key, operator, value)
+	case FloatEquals,
+		FloatGreaterThan,
+		FloatGreaterThanOrEqual,
+		FloatLessThan,
+		FloatLessThanOrEqual:
+		return newFloatCondition(key, operator, value)
+	case SameAs,
+		Contains,
+		StartsWith,
+		EndsWith:
+		return newStringCondition(key, operator, value)
+	case In:
+		return newStringSliceCondition(key, operator, value)
+	case Matches:
+		return newRegexCondition(key, operator, value)
+	case Is:
+		return newBoolCondition(key, operator, value)
+	case Exists:
+		return newExistsCondition(key, operator)
+	default:
+		return newErrorCondition(fmt.Errorf("no operator with ID %d", operator))
+	}
+}
diff --git a/database/query/fetcher.go b/database/query/fetcher.go
new file mode 100644
index 0000000..c2004bd
--- /dev/null
+++ b/database/query/fetcher.go
@@ -0,0 +1,78 @@
+package query
+
+import (
+	"github.com/tidwall/gjson"
+)
+
+const (
+	emptyString = ""
+)
+
+// Fetcher provides an interface to supply the query matcher a method to retrieve values from an object.
+type Fetcher interface {
+	GetString(key string) (value string, ok bool)
+	GetInt(key string) (value int64, ok bool)
+	GetFloat(key string) (value float64, ok bool)
+	GetBool(key string) (value bool, ok bool)
+	Exists(key string) bool
+}
+
+// JSONFetcher is a json string with get functions.
+type JSONFetcher struct {
+	json string
+}
+
+// NewJSONFetcher adds the Fetcher interface to a JSON string.
+func NewJSONFetcher(json string) *JSONFetcher {
+	return &JSONFetcher{
+		json: json,
+	}
+}
+
+// GetString returns the string found by the given json key and whether it could be successfully extracted.
+func (jf *JSONFetcher) GetString(key string) (value string, ok bool) {
+	result := gjson.Get(jf.json, key)
+	if !result.Exists() || result.Type != gjson.String {
+		return emptyString, false
+	}
+	return result.String(), true
+}
+
+// GetInt returns the int found by the given json key and whether it could be successfully extracted.
+func (jf *JSONFetcher) GetInt(key string) (value int64, ok bool) {
+	result := gjson.Get(jf.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Int(), true
+}
+
+// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
+func (jf *JSONFetcher) GetFloat(key string) (value float64, ok bool) {
+	result := gjson.Get(jf.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Float(), true
+}
+
+// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
+func (jf *JSONFetcher) GetBool(key string) (value bool, ok bool) {
+	result := gjson.Get(jf.json, key)
+	switch {
+	case !result.Exists():
+		return false, false
+	case result.Type == gjson.True:
+		return true, true
+	case result.Type == gjson.False:
+		return false, true
+	default:
+		return false, false
+	}
+}
+
+// Exists returns the whether the given key exists.
+func (jf *JSONFetcher) Exists(key string) bool {
+	result := gjson.Get(jf.json, key)
+	return result.Exists()
+}
diff --git a/database/query/parser.go b/database/query/parser.go
new file mode 100644
index 0000000..3fcf1ae
--- /dev/null
+++ b/database/query/parser.go
@@ -0,0 +1,44 @@
+package query
+
+var (
+	operatorNames = map[string]uint8{
+		"==":         Equals,
+		">":          GreaterThan,
+		">=":         GreaterThanOrEqual,
+		"<":          LessThan,
+		"<=":         LessThanOrEqual,
+		"f==":        FloatEquals,
+		"f>":         FloatGreaterThan,
+		"f>=":        FloatGreaterThanOrEqual,
+		"f<":         FloatLessThan,
+		"f<=":        FloatLessThanOrEqual,
+		"sameas":     SameAs,
+		"s==":        SameAs,
+		"contains":   Contains,
+		"co":         Contains,
+		"startswith": StartsWith,
+		"sw":         StartsWith,
+		"endswith":   EndsWith,
+		"ew":         EndsWith,
+		"in":         In,
+		"matches":    Matches,
+		"re":         Matches,
+		"is":         Is,
+		"exists":     Exists,
+		"ex":         Exists,
+	}
+)
+
+func getOpName(operator uint8) string {
+	for opName, op := range operatorNames {
+		if op == operator {
+			return opName
+		}
+	}
+	return "[unknown]"
+}
+
+// ParseQuery parses a plaintext query.
+func ParseQuery(query string) (*Query, error) {
+	return nil, nil
+}
diff --git a/database/query/query.go b/database/query/query.go
new file mode 100644
index 0000000..8924ac7
--- /dev/null
+++ b/database/query/query.go
@@ -0,0 +1,98 @@
+package query
+
+import (
+	"fmt"
+	"regexp"
+)
+
+// Example:
+// q.New("core:/",
+//   q.Where("a", q.GreaterThan, 0),
+//   q.Where("b", q.Equals, 0),
+//   q.Or(
+//       q.Where("c", q.StartsWith, "x"),
+//       q.Where("d", q.Contains, "y")
+//     )
+//   )
+
+var (
+	prefixExpr = regexp.MustCompile("^[a-z-]+:")
+)
+
+// Query contains a compiled query.
+type Query struct {
+	prefix     string
+	conditions []Condition
+}
+
+// New creates a new query.
+func New(prefix string, conditions ...Condition) (*Query, error) {
+	// check prefix
+	if !prefixExpr.MatchString(prefix) {
+		return nil, fmt.Errorf("invalid prefix: %s", prefix)
+	}
+
+	// check conditions
+	var err error
+	for _, cond := range conditions {
+		err = cond.check()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// return query
+	return &Query{
+		prefix:     prefix,
+		conditions: conditions,
+	}, nil
+}
+
+// MustCompile creates a new query and panics on an error.
+func MustCompile(prefix string, conditions ...Condition) *Query {
+	q, err := New(prefix, conditions...)
+	if err != nil {
+		panic(err)
+	}
+	return q
+}
+
+// Prepend prepends (check first) new query conditions to the query.
+func (q *Query) Prepend(conditions ...Condition) error {
+	// check conditions
+	var err error
+	for _, cond := range conditions {
+		err = cond.check()
+		if err != nil {
+			return err
+		}
+	}
+
+	q.conditions = append(conditions, q.conditions...)
+	return nil
+}
+
+// Append appends (check last) new query conditions to the query.
+func (q *Query) Append(conditions ...Condition) error {
+	// check conditions
+	var err error
+	for _, cond := range conditions {
+		err = cond.check()
+		if err != nil {
+			return err
+		}
+	}
+
+	q.conditions = append(q.conditions, conditions...)
+	return nil
+}
+
+// Matches checks whether the query matches the supplied data object.
+func (q *Query) Matches(f Fetcher) bool {
+	for _, cond := range q.conditions {
+		if !cond.complies(f) {
+			return false
+		}
+	}
+	return true
+}
diff --git a/database/query/query_test.go b/database/query/query_test.go
new file mode 100644
index 0000000..a97fe2d
--- /dev/null
+++ b/database/query/query_test.go
@@ -0,0 +1,104 @@
+package query
+
+import (
+	"testing"
+)
+
+var (
+	// copied from https://github.com/tidwall/gjson/blob/master/gjson_test.go
+	testJson = `{"age":100, "name":{"here":"B\\\"R"},
+  "noop":{"what is a wren?":"a bird"},
+  "happy":true,"immortal":false,
+  "items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7],
+  "arr":["1",2,"3",{"hello":"world"},"4",5],
+  "vals":[1,2,3,{"sadf":sdf"asdf"}],"name":{"first":"tom","last":null},
+  "created":"2014-05-16T08:28:06.989Z",
+  "loggy":{
+  	"programmers": [
+    	    {
+    	        "firstName": "Brett",
+    	        "lastName": "McLaughlin",
+    	        "email": "aaaa",
+  			"tag": "good"
+    	    },
+    	    {
+    	        "firstName": "Jason",
+    	        "lastName": "Hunter",
+    	        "email": "bbbb",
+  			"tag": "bad"
+    	    },
+    	    {
+    	        "firstName": "Elliotte",
+    	        "lastName": "Harold",
+    	        "email": "cccc",
+  			"tag":, "good"
+    	    },
+  		{
+  			"firstName": 1002.3,
+  			"age": 101
+  		}
+    	]
+  },
+  "lastly":{"yay":"final"},
+	"temperature": 120.413
+}`
+)
+
+func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) {
+	q := MustCompile("test:")
+	err := q.Append(condition)
+	if err != nil {
+		t.Errorf("append failed: %s", err)
+	}
+
+	matched := q.Matches(f)
+	switch {
+	case matched && !shouldMatch:
+		t.Errorf("query should match")
+	case !matched && shouldMatch:
+		t.Errorf("query should not match")
+	}
+}
+
+func TestQuery(t *testing.T) {
+
+	// if !gjson.Valid(testJson) {
+	// 	t.Fatal("test json is invalid")
+	// }
+	f := NewJSONFetcher(testJson)
+
+	testQuery(t, f, true, Where("age", Equals, 100))
+	testQuery(t, f, true, Where("age", GreaterThan, uint8(99)))
+	testQuery(t, f, true, Where("age", GreaterThanOrEqual, 99))
+	testQuery(t, f, true, Where("age", GreaterThanOrEqual, 100))
+	testQuery(t, f, true, Where("age", LessThan, 101))
+	testQuery(t, f, true, Where("age", LessThanOrEqual, "101"))
+	testQuery(t, f, true, Where("age", LessThanOrEqual, 100))
+
+	testQuery(t, f, true, Where("temperature", FloatEquals, 120.413))
+	testQuery(t, f, true, Where("temperature", FloatGreaterThan, 120))
+	testQuery(t, f, true, Where("temperature", FloatGreaterThanOrEqual, 120))
+	testQuery(t, f, true, Where("temperature", FloatGreaterThanOrEqual, 120.413))
+	testQuery(t, f, true, Where("temperature", FloatLessThan, 121))
+	testQuery(t, f, true, Where("temperature", FloatLessThanOrEqual, "121"))
+	testQuery(t, f, true, Where("temperature", FloatLessThanOrEqual, "120.413"))
+
+	testQuery(t, f, true, Where("lastly.yay", Matches, "final"))
+	testQuery(t, f, true, Where("lastly.yay", Contains, "ina"))
+	testQuery(t, f, true, Where("lastly.yay", StartsWith, "fin"))
+	testQuery(t, f, true, Where("lastly.yay", EndsWith, "nal"))
+	testQuery(t, f, true, Where("lastly.yay", In, "draft,final"))
+	testQuery(t, f, true, Where("lastly.yay", In, "final,draft"))
+
+	testQuery(t, f, true, Where("happy", Is, "true"))
+	testQuery(t, f, true, Where("happy", Is, "t"))
+	testQuery(t, f, true, Where("happy", Is, "1"))
+	testQuery(t, f, true, Not(Where("happy", Is, "false")))
+	testQuery(t, f, true, Not(Where("happy", Is, "f")))
+	testQuery(t, f, true, Not(Where("happy", Is, "0")))
+
+	testQuery(t, f, true, Where("happy", Exists, nil))
+
+	testQuery(t, f, true, Where("created", Matches, "^2014-[0-9]{2}-[0-9]{2}T"))
+
+}

From 115b18dfb659a683ec155d8ba3131516a4adb90d Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 30 Aug 2018 19:03:45 +0200
Subject: [PATCH 02/20] Add first part of query parser / Finish query building

---
 database/query/condition-and.go         |  13 ++
 database/query/condition-bool.go        |   4 +
 database/query/condition-error.go       |   4 +
 database/query/condition-exists.go      |   5 +
 database/query/condition-float.go       |   4 +
 database/query/condition-int.go         |   4 +
 database/query/condition-no.go          |  16 ++
 database/query/condition-not.go         |  14 ++
 database/query/condition-or.go          |  13 ++
 database/query/condition-regex.go       |   4 +
 database/query/condition-string.go      |   6 +-
 database/query/condition-stringslice.go |   4 +
 database/query/condition.go             |   2 +-
 database/query/parser.go                | 193 +++++++++++++++++++++++-
 database/query/parser_test.go           |  18 +++
 database/query/query.go                 |  73 ++++-----
 database/query/query_test.go            |  26 ++--
 17 files changed, 337 insertions(+), 66 deletions(-)
 create mode 100644 database/query/condition-no.go
 create mode 100644 database/query/parser_test.go

diff --git a/database/query/condition-and.go b/database/query/condition-and.go
index 913cbcf..2346b78 100644
--- a/database/query/condition-and.go
+++ b/database/query/condition-and.go
@@ -1,5 +1,10 @@
 package query
 
+import (
+	"fmt"
+	"strings"
+)
+
 // And combines multiple conditions with a logical _AND_ operator.
 func And(conditions ...Condition) Condition {
 	return &andCond{
@@ -29,3 +34,11 @@ func (c *andCond) check() (err error) {
 	}
 	return nil
 }
+
+func (c *andCond) string() string {
+	var all []string
+	for _, cond := range c.conditions {
+		all = append(all, cond.string())
+	}
+	return fmt.Sprintf("(%s)", strings.Join(all, " and "))
+}
diff --git a/database/query/condition-bool.go b/database/query/condition-bool.go
index 2e6bec8..49f1608 100644
--- a/database/query/condition-bool.go
+++ b/database/query/condition-bool.go
@@ -62,3 +62,7 @@ func (c *boolCondition) check() error {
 	}
 	return nil
 }
+
+func (c *boolCondition) string() string {
+	return fmt.Sprintf("%s %s %t", c.key, getOpName(c.operator), c.value)
+}
diff --git a/database/query/condition-error.go b/database/query/condition-error.go
index a8c7610..ab5b405 100644
--- a/database/query/condition-error.go
+++ b/database/query/condition-error.go
@@ -17,3 +17,7 @@ func (c *errorCondition) complies(f Fetcher) bool {
 func (c *errorCondition) check() error {
 	return c.err
 }
+
+func (c *errorCondition) string() string {
+	return "[ERROR]"
+}
diff --git a/database/query/condition-exists.go b/database/query/condition-exists.go
index 086f200..bde67a8 100644
--- a/database/query/condition-exists.go
+++ b/database/query/condition-exists.go
@@ -2,6 +2,7 @@ package query
 
 import (
 	"errors"
+	"fmt"
 )
 
 type existsCondition struct {
@@ -26,3 +27,7 @@ func (c *existsCondition) check() error {
 	}
 	return nil
 }
+
+func (c *existsCondition) string() string {
+	return fmt.Sprintf("%s %s", c.key, getOpName(c.operator))
+}
diff --git a/database/query/condition-float.go b/database/query/condition-float.go
index 3627b91..851b3f2 100644
--- a/database/query/condition-float.go
+++ b/database/query/condition-float.go
@@ -88,3 +88,7 @@ func (c *floatCondition) check() error {
 	}
 	return nil
 }
+
+func (c *floatCondition) string() string {
+	return fmt.Sprintf("%s %s %f", c.key, getOpName(c.operator), c.value)
+}
diff --git a/database/query/condition-int.go b/database/query/condition-int.go
index b169a12..eaf427b 100644
--- a/database/query/condition-int.go
+++ b/database/query/condition-int.go
@@ -84,3 +84,7 @@ func (c *intCondition) check() error {
 	}
 	return nil
 }
+
+func (c *intCondition) string() string {
+	return fmt.Sprintf("%s %s %d", c.key, getOpName(c.operator), c.value)
+}
diff --git a/database/query/condition-no.go b/database/query/condition-no.go
new file mode 100644
index 0000000..ff72fcc
--- /dev/null
+++ b/database/query/condition-no.go
@@ -0,0 +1,16 @@
+package query
+
+type noCond struct {
+}
+
+func (c *noCond) complies(f Fetcher) bool {
+	return true
+}
+
+func (c *noCond) check() (err error) {
+	return nil
+}
+
+func (c *noCond) string() string {
+	return ""
+}
diff --git a/database/query/condition-not.go b/database/query/condition-not.go
index 7249568..d395bc9 100644
--- a/database/query/condition-not.go
+++ b/database/query/condition-not.go
@@ -1,5 +1,10 @@
 package query
 
+import (
+	"fmt"
+	"strings"
+)
+
 // Not negates the supplied condition.
 func Not(c Condition) Condition {
 	return &notCond{
@@ -18,3 +23,12 @@ func (c *notCond) complies(f Fetcher) bool {
 func (c *notCond) check() error {
 	return c.notC.check()
 }
+
+func (c *notCond) string() string {
+	next := c.notC.string()
+	if strings.HasPrefix(next, "(") {
+		return fmt.Sprintf("not %s", c.notC.string())
+	}
+	splitted := strings.Split(next, " ")
+	return strings.Join(append([]string{splitted[0], "not"}, splitted[1:]...), " ")
+}
diff --git a/database/query/condition-or.go b/database/query/condition-or.go
index e93228b..d790f48 100644
--- a/database/query/condition-or.go
+++ b/database/query/condition-or.go
@@ -1,5 +1,10 @@
 package query
 
+import (
+	"fmt"
+	"strings"
+)
+
 // Or combines multiple conditions with a logical _OR_ operator.
 func Or(conditions ...Condition) Condition {
 	return &orCond{
@@ -29,3 +34,11 @@ func (c *orCond) check() (err error) {
 	}
 	return nil
 }
+
+func (c *orCond) string() string {
+	var all []string
+	for _, cond := range c.conditions {
+		all = append(all, cond.string())
+	}
+	return fmt.Sprintf("(%s)", strings.Join(all, " or "))
+}
diff --git a/database/query/condition-regex.go b/database/query/condition-regex.go
index b32d212..b6c2e8b 100644
--- a/database/query/condition-regex.go
+++ b/database/query/condition-regex.go
@@ -55,3 +55,7 @@ func (c *regexCondition) check() error {
 	}
 	return nil
 }
+
+func (c *regexCondition) string() string {
+	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.regex.String())
+}
diff --git a/database/query/condition-string.go b/database/query/condition-string.go
index f255693..8890b3c 100644
--- a/database/query/condition-string.go
+++ b/database/query/condition-string.go
@@ -35,7 +35,7 @@ func (c *stringCondition) complies(f Fetcher) bool {
 	}
 
 	switch c.operator {
-	case Matches:
+	case SameAs:
 		return c.value == comp
 	case Contains:
 		return strings.Contains(comp, c.value)
@@ -54,3 +54,7 @@ func (c *stringCondition) check() error {
 	}
 	return nil
 }
+
+func (c *stringCondition) string() string {
+	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.value)
+}
diff --git a/database/query/condition-stringslice.go b/database/query/condition-stringslice.go
index cfd04d4..f8597cc 100644
--- a/database/query/condition-stringslice.go
+++ b/database/query/condition-stringslice.go
@@ -58,3 +58,7 @@ func (c *stringSliceCondition) check() error {
 	}
 	return nil
 }
+
+func (c *stringSliceCondition) string() string {
+	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), strings.Join(c.value, ","))
+}
diff --git a/database/query/condition.go b/database/query/condition.go
index 85ae28b..3c5e7d2 100644
--- a/database/query/condition.go
+++ b/database/query/condition.go
@@ -6,7 +6,7 @@ import "fmt"
 type Condition interface {
 	complies(f Fetcher) bool
 	check() error
-	// string() string
+	string() string
 }
 
 // Operators
diff --git a/database/query/parser.go b/database/query/parser.go
index 3fcf1ae..2ceebd9 100644
--- a/database/query/parser.go
+++ b/database/query/parser.go
@@ -1,5 +1,12 @@
 package query
 
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
 var (
 	operatorNames = map[string]uint8{
 		"==":         Equals,
@@ -27,18 +34,194 @@ var (
 		"exists":     Exists,
 		"ex":         Exists,
 	}
+
+	primaryNames = make(map[uint8]string)
 )
 
-func getOpName(operator uint8) string {
-	for opName, op := range operatorNames {
-		if op == operator {
-			return opName
+func init() {
+	for opName, opID := range operatorNames {
+		name, ok := primaryNames[opID]
+		if ok {
+			if len(name) < len(opName) {
+				primaryNames[opID] = opName
+			}
+		} else {
+			primaryNames[opID] = opName
 		}
 	}
+}
+
+func getOpName(operator uint8) string {
+	name, ok := primaryNames[operator]
+	if ok {
+		return name
+	}
 	return "[unknown]"
 }
 
-// ParseQuery parses a plaintext query.
+type treeElement struct {
+	branches       []*treeElement
+	text           string
+	globalPosition int
+}
+
+var (
+	escapeReplacer = regexp.MustCompile("\\\\([^\\\\])")
+)
+
+// prepToken removes surrounding parenthesis and escape characters.
+func prepToken(text string) string {
+	return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
+}
+
+func extractSnippets(text string) (snippets []*treeElement, err error) {
+
+	skip := false
+	start := -1
+	inParenthesis := false
+	var pos int
+	var char rune
+
+	for pos, char = range text {
+
+		// skip
+		if skip {
+			skip = false
+			continue
+		}
+		if char == '\\' {
+			skip = true
+		}
+
+		// wait for parenthesis to be over
+		if inParenthesis {
+			if char == '"' {
+				snippets = append(snippets, &treeElement{
+					text:           prepToken(text[start+1 : pos]),
+					globalPosition: start + 1,
+				})
+				start = -1
+				inParenthesis = false
+			}
+			continue
+		}
+
+		// handle segments
+		switch char {
+		case '\t', '\n', '\r', ' ', '(', ')':
+			if start >= 0 {
+				snippets = append(snippets, &treeElement{
+					text:           prepToken(text[start:pos]),
+					globalPosition: start,
+				})
+				start = -1
+			}
+		default:
+			if start == -1 {
+				start = pos
+			}
+		}
+
+		// handle special segment characters
+		switch char {
+		case '(', ')':
+			snippets = append(snippets, &treeElement{
+				text:           text[pos : pos+1],
+				globalPosition: pos,
+			})
+		case '"':
+			if start < pos {
+				return nil, fmt.Errorf("parenthesis ('\"') may not be within words, please escape with '\\' (position: %d)", pos+1)
+			}
+			inParenthesis = true
+		}
+
+	}
+
+	// add last
+	snippets = append(snippets, &treeElement{
+		text:           prepToken(text[start : pos+1]),
+		globalPosition: start,
+	})
+
+	return snippets, nil
+
+}
+
+// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
 func ParseQuery(query string) (*Query, error) {
+	snippets, err := extractSnippets(query)
+	if err != nil {
+		return nil, err
+	}
+	snippetsPos := 0
+
+	getElement := func() (*treeElement, error) {
+		if snippetsPos >= len(snippets) {
+			return nil, fmt.Errorf("unexpected end at position %d", len(query))
+		}
+		return snippets[snippetsPos], nil
+	}
+
+	// check for query word
+	queryWord, err := getElement()
+	if err != nil {
+		return nil, err
+	}
+	if queryWord.text != "query" {
+		return nil, errors.New("queries must start with \"query\"")
+	}
+
+	// get prefix
+	prefix, err := getElement()
+	if err != nil {
+		return nil, err
+	}
+
+	// check if no condition
+	if len(snippets) == 2 {
+		return New(prefix.text, nil)
+	}
+
+	// check for where word
+	whereWord, err := getElement()
+	if err != nil {
+		return nil, err
+	}
+	if whereWord.text != "where" {
+		return nil, errors.New("filtering queries must start conditions with \"where\"")
+	}
+
+	// parse conditions
+	condition, err := parseCondition(getElement)
+	if err != nil {
+		return nil, err
+	}
+
+	// check for additional tokens
+	// token := s.Scan()
+	// if token != scanner.EOF {
+	// 	return nil, fmt.Errorf("unexpected additional tokens at position %d", s.Position)
+	// }
+
+	return New(prefix.text, condition)
+}
+
+func parseCondition(getElement func() (*treeElement, error)) (Condition, error) {
+	first, err := getElement()
+	if err != nil {
+		return nil, err
+	}
+
+	switch first.text {
+	case "(":
+		return parseAndOr(getElement, true, nil, false)
+		// case ""
+	}
+
+	return nil, nil
+}
+
+func parseAndOr(getElement func() (*treeElement, error), expectBracket bool, preParsedCondition Condition, preParsedIsOr bool) (Condition, error) {
 	return nil, nil
 }
diff --git a/database/query/parser_test.go b/database/query/parser_test.go
new file mode 100644
index 0000000..53cbd31
--- /dev/null
+++ b/database/query/parser_test.go
@@ -0,0 +1,18 @@
+package query
+
+import (
+	"testing"
+)
+
+func TestMakeTree(t *testing.T) {
+	text1 := `(age > 100 and experience <= "99") or (age < 10 and motivation > 50) or name matches Mr.\"X or name matches y`
+	snippets, err := extractSnippets(text1)
+	if err != nil {
+		t.Errorf("failed to make tree: %s", err)
+	} else {
+		for _, el := range snippets {
+			t.Errorf("%+v", el)
+		}
+	}
+	// t.Error(spew.Sprintf("%v", treeElement))
+}
diff --git a/database/query/query.go b/database/query/query.go
index 8924ac7..c8fbf1e 100644
--- a/database/query/query.go
+++ b/database/query/query.go
@@ -3,6 +3,7 @@ package query
 import (
 	"fmt"
 	"regexp"
+	"strings"
 )
 
 // Example:
@@ -21,78 +22,56 @@ var (
 
 // Query contains a compiled query.
 type Query struct {
-	prefix     string
-	conditions []Condition
+	prefix    string
+	condition Condition
 }
 
 // New creates a new query.
-func New(prefix string, conditions ...Condition) (*Query, error) {
+func New(prefix string, condition Condition) (*Query, error) {
 	// check prefix
 	if !prefixExpr.MatchString(prefix) {
 		return nil, fmt.Errorf("invalid prefix: %s", prefix)
 	}
 
-	// check conditions
-	var err error
-	for _, cond := range conditions {
-		err = cond.check()
+	// check condition
+	if condition != nil {
+		err := condition.check()
 		if err != nil {
 			return nil, err
 		}
+	} else {
+		condition = &noCond{}
 	}
 
 	// return query
 	return &Query{
-		prefix:     prefix,
-		conditions: conditions,
+		prefix:    prefix,
+		condition: condition,
 	}, nil
 }
 
 // MustCompile creates a new query and panics on an error.
-func MustCompile(prefix string, conditions ...Condition) *Query {
-	q, err := New(prefix, conditions...)
+func MustCompile(prefix string, condition Condition) *Query {
+	q, err := New(prefix, condition)
 	if err != nil {
 		panic(err)
 	}
 	return q
 }
 
-// Prepend prepends (check first) new query conditions to the query.
-func (q *Query) Prepend(conditions ...Condition) error {
-	// check conditions
-	var err error
-	for _, cond := range conditions {
-		err = cond.check()
-		if err != nil {
-			return err
-		}
-	}
-
-	q.conditions = append(conditions, q.conditions...)
-	return nil
-}
-
-// Append appends (check last) new query conditions to the query.
-func (q *Query) Append(conditions ...Condition) error {
-	// check conditions
-	var err error
-	for _, cond := range conditions {
-		err = cond.check()
-		if err != nil {
-			return err
-		}
-	}
-
-	q.conditions = append(q.conditions, conditions...)
-	return nil
-}
-
 // Matches checks whether the query matches the supplied data object.
 func (q *Query) Matches(f Fetcher) bool {
-	for _, cond := range q.conditions {
-		if !cond.complies(f) {
-			return false
-		}
-	}
-	return true
+	return q.condition.complies(f)
+}
+
+// String returns the string representation of the query.
+func (q *Query) String() string {
+	text := q.condition.string()
+	if text == "" {
+		return fmt.Sprintf("query %s", q.prefix)
+	}
+	if strings.HasPrefix(text, "(") {
+		text = text[1 : len(text)-1]
+	}
+	return fmt.Sprintf("query %s where %s", q.prefix, text)
 }
diff --git a/database/query/query_test.go b/database/query/query_test.go
index a97fe2d..39ce970 100644
--- a/database/query/query_test.go
+++ b/database/query/query_test.go
@@ -45,18 +45,16 @@ var (
 )
 
 func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) {
-	q := MustCompile("test:")
-	err := q.Append(condition)
-	if err != nil {
-		t.Errorf("append failed: %s", err)
-	}
+	q := MustCompile("test:", condition)
+
+	// fmt.Printf("%s\n", q.String())
 
 	matched := q.Matches(f)
 	switch {
-	case matched && !shouldMatch:
-		t.Errorf("query should match")
 	case !matched && shouldMatch:
-		t.Errorf("query should not match")
+		t.Errorf("should match: %s", q.String())
+	case matched && !shouldMatch:
+		t.Errorf("should not match: %s", q.String())
 	}
 }
 
@@ -83,7 +81,7 @@ func TestQuery(t *testing.T) {
 	testQuery(t, f, true, Where("temperature", FloatLessThanOrEqual, "121"))
 	testQuery(t, f, true, Where("temperature", FloatLessThanOrEqual, "120.413"))
 
-	testQuery(t, f, true, Where("lastly.yay", Matches, "final"))
+	testQuery(t, f, true, Where("lastly.yay", SameAs, "final"))
 	testQuery(t, f, true, Where("lastly.yay", Contains, "ina"))
 	testQuery(t, f, true, Where("lastly.yay", StartsWith, "fin"))
 	testQuery(t, f, true, Where("lastly.yay", EndsWith, "nal"))
@@ -92,10 +90,14 @@ func TestQuery(t *testing.T) {
 
 	testQuery(t, f, true, Where("happy", Is, "true"))
 	testQuery(t, f, true, Where("happy", Is, "t"))
-	testQuery(t, f, true, Where("happy", Is, "1"))
-	testQuery(t, f, true, Not(Where("happy", Is, "false")))
-	testQuery(t, f, true, Not(Where("happy", Is, "f")))
 	testQuery(t, f, true, Not(Where("happy", Is, "0")))
+	testQuery(t, f, true, And(
+		Where("happy", Is, "1"),
+		Not(Or(
+			Where("happy", Is, false),
+			Where("happy", Is, "f"),
+		)),
+	))
 
 	testQuery(t, f, true, Where("happy", Exists, nil))
 

From e40d66e103075da554975019014bdf46609aeef8 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 31 Aug 2018 17:11:59 +0200
Subject: [PATCH 03/20] Finish query package for now

---
 database/query/README.md                |  55 ++++
 database/query/condition-bool.go        |   2 +-
 database/query/condition-exists.go      |   2 +-
 database/query/condition-float.go       |   4 +-
 database/query/condition-int.go         |   4 +-
 database/query/condition-regex.go       |   2 +-
 database/query/condition-string.go      |   2 +-
 database/query/condition-stringslice.go |  10 +-
 database/query/condition_test.go        |  76 +++++
 database/query/operators.go             |  53 ++++
 database/query/operators_test.go        |   9 +
 database/query/parser.go                | 388 ++++++++++++++++--------
 database/query/parser_test.go           | 158 +++++++++-
 database/query/query.go                 | 117 +++++--
 database/query/query_test.go            |  13 +-
 15 files changed, 714 insertions(+), 181 deletions(-)
 create mode 100644 database/query/README.md
 create mode 100644 database/query/condition_test.go
 create mode 100644 database/query/operators.go
 create mode 100644 database/query/operators_test.go

diff --git a/database/query/README.md b/database/query/README.md
new file mode 100644
index 0000000..9311417
--- /dev/null
+++ b/database/query/README.md
@@ -0,0 +1,55 @@
+# Query
+
+## Control Flow
+
+- Grouping with `(` and `)`
+- Chaining with `and` and `or`
+  - _NO_ mixing! Be explicit and use grouping.
+- Negation with `not`
+  - in front of expression for group: `not (...)`
+  - inside expression for clause: `name not matches "^King "`
+
+## Selectors
+
+Supported by all feeders:
+- root level field: `field`
+- sub level field: `field.sub`
+- array/slice/map access: `map.0`
+- array/slice/map length: `map.#`
+
+Please note that some feeders may have other special characters. It is advised to only use alphanumeric characters for keys.
+
+## Operators
+
+| Name | Textual | Req. Type | Internal Type | Compared with |
+|---|---|---|---|
+| Equals | `==` | int | int64 | `==` |
+| GreaterThan | `>` | int | int64 | `>` |
+| GreaterThanOrEqual | `>=` | int | int64 | `>=` |
+| LessThan | `<`  | int | int64 | `<` |
+| LessThanOrEqual | `<=` | int | int64 | `<=` |
+| FloatEquals | `f==` | float | float64 | `==` |
+| FloatGreaterThan | `f>` | float | float64 | `>` |
+| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` |
+| FloatLessThan | `f<` | float | float64 | `<` |
+| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` |
+| SameAs | `sameas`, `s==` | string | string | `==` |
+| Contains | `contains`, `co` | string | string | `strings.Contains()` |
+| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` |
+| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` |
+| In | `in` | string | string | for loop with `==` |
+| Matches | `matches`, `re` | string | int64 | `regexp.Regexp.Matches()` |
+| Is | `is` | bool* | bool | `==` |
+| Exists | `exists`, `ex` | any | n/a | n/a |
+
+\*accepts strings: 1, t, T, TRUE, true, True, 0, f, F, FALSE
+
+## Escaping
+
+If you need to use a control character within a value (ie. not for controlling), escape it with `\`.
+It is recommended to wrap a word into parenthesis instead of escaping control characters, when possible.
+
+| Location | Characters to be escaped |
+|---|---|
+| Within parenthesis (`"`) | `"`, `\` |
+| Everywhere else | `(`, `)`, `"`, `\`, `\t`, `\r`, `\n`, ` ` (space) |
diff --git a/database/query/condition-bool.go b/database/query/condition-bool.go
index 49f1608..e9d7c2b 100644
--- a/database/query/condition-bool.go
+++ b/database/query/condition-bool.go
@@ -64,5 +64,5 @@ func (c *boolCondition) check() error {
 }
 
 func (c *boolCondition) string() string {
-	return fmt.Sprintf("%s %s %t", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %t", escapeString(c.key), getOpName(c.operator), c.value)
 }
diff --git a/database/query/condition-exists.go b/database/query/condition-exists.go
index bde67a8..2e2b013 100644
--- a/database/query/condition-exists.go
+++ b/database/query/condition-exists.go
@@ -29,5 +29,5 @@ func (c *existsCondition) check() error {
 }
 
 func (c *existsCondition) string() string {
-	return fmt.Sprintf("%s %s", c.key, getOpName(c.operator))
+	return fmt.Sprintf("%s %s", escapeString(c.key), getOpName(c.operator))
 }
diff --git a/database/query/condition-float.go b/database/query/condition-float.go
index 851b3f2..c34cf7f 100644
--- a/database/query/condition-float.go
+++ b/database/query/condition-float.go
@@ -31,6 +31,8 @@ func newFloatCondition(key string, operator uint8, value interface{}) *floatCond
 		parsedValue = float64(v)
 	case uint8:
 		parsedValue = float64(v)
+	case uint16:
+		parsedValue = float64(v)
 	case uint32:
 		parsedValue = float64(v)
 	case float32:
@@ -90,5 +92,5 @@ func (c *floatCondition) check() error {
 }
 
 func (c *floatCondition) string() string {
-	return fmt.Sprintf("%s %s %f", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %g", escapeString(c.key), getOpName(c.operator), c.value)
 }
diff --git a/database/query/condition-int.go b/database/query/condition-int.go
index eaf427b..8f18ab2 100644
--- a/database/query/condition-int.go
+++ b/database/query/condition-int.go
@@ -31,6 +31,8 @@ func newIntCondition(key string, operator uint8, value interface{}) *intConditio
 		parsedValue = int64(v)
 	case uint8:
 		parsedValue = int64(v)
+	case uint16:
+		parsedValue = int64(v)
 	case uint32:
 		parsedValue = int64(v)
 	case string:
@@ -86,5 +88,5 @@ func (c *intCondition) check() error {
 }
 
 func (c *intCondition) string() string {
-	return fmt.Sprintf("%s %s %d", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %d", escapeString(c.key), getOpName(c.operator), c.value)
 }
diff --git a/database/query/condition-regex.go b/database/query/condition-regex.go
index b6c2e8b..d795c84 100644
--- a/database/query/condition-regex.go
+++ b/database/query/condition-regex.go
@@ -57,5 +57,5 @@ func (c *regexCondition) check() error {
 }
 
 func (c *regexCondition) string() string {
-	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.regex.String())
+	return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.regex.String()))
 }
diff --git a/database/query/condition-string.go b/database/query/condition-string.go
index 8890b3c..747e337 100644
--- a/database/query/condition-string.go
+++ b/database/query/condition-string.go
@@ -56,5 +56,5 @@ func (c *stringCondition) check() error {
 }
 
 func (c *stringCondition) string() string {
-	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.value))
 }
diff --git a/database/query/condition-stringslice.go b/database/query/condition-stringslice.go
index f8597cc..ab3004d 100644
--- a/database/query/condition-stringslice.go
+++ b/database/query/condition-stringslice.go
@@ -20,7 +20,7 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
 		parsedValue := strings.Split(v, ",")
 		if len(parsedValue) < 2 {
 			return &stringSliceCondition{
-				key:      fmt.Sprintf("could not parse \"%s\" to []string", v),
+				key:      v,
 				operator: errorPresent,
 			}
 		}
@@ -29,6 +29,12 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
 			operator: operator,
 			value:    parsedValue,
 		}
+	case []string:
+		return &stringSliceCondition{
+			key:      key,
+			operator: operator,
+			value:    v,
+		}
 	default:
 		return &stringSliceCondition{
 			key:      fmt.Sprintf("incompatible value %v for []string", value),
@@ -60,5 +66,5 @@ func (c *stringSliceCondition) check() error {
 }
 
 func (c *stringSliceCondition) string() string {
-	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), strings.Join(c.value, ","))
+	return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(strings.Join(c.value, ",")))
 }
diff --git a/database/query/condition_test.go b/database/query/condition_test.go
new file mode 100644
index 0000000..eb871a7
--- /dev/null
+++ b/database/query/condition_test.go
@@ -0,0 +1,76 @@
+package query
+
+import "testing"
+
+func testSuccess(t *testing.T, c Condition) {
+	err := c.check()
+	if err != nil {
+		t.Errorf("failed: %s", err)
+	}
+}
+
+func TestInterfaces(t *testing.T) {
+	testSuccess(t, newIntCondition("banana", Equals, uint(1)))
+	testSuccess(t, newIntCondition("banana", Equals, uint8(1)))
+	testSuccess(t, newIntCondition("banana", Equals, uint16(1)))
+	testSuccess(t, newIntCondition("banana", Equals, uint32(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int8(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int16(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int32(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int64(1)))
+	testSuccess(t, newIntCondition("banana", Equals, "1"))
+
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint8(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint16(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint32(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int8(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int16(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int32(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int64(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, float32(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, float64(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, "1.1"))
+
+	testSuccess(t, newStringCondition("banana", SameAs, "coconut"))
+	testSuccess(t, newRegexCondition("banana", Matches, "coconut"))
+	testSuccess(t, newStringSliceCondition("banana", FloatEquals, []string{"banana", "coconut"}))
+	testSuccess(t, newStringSliceCondition("banana", FloatEquals, "banana,coconut"))
+}
+
+func testCondError(t *testing.T, c Condition) {
+	err := c.check()
+	if err == nil {
+		t.Error("should fail")
+	}
+}
+
+func TestConditionErrors(t *testing.T) {
+	// test invalid value types
+	testCondError(t, newBoolCondition("banana", Is, 1))
+	testCondError(t, newFloatCondition("banana", FloatEquals, true))
+	testCondError(t, newIntCondition("banana", Equals, true))
+	testCondError(t, newStringCondition("banana", SameAs, 1))
+	testCondError(t, newRegexCondition("banana", Matches, 1))
+	testCondError(t, newStringSliceCondition("banana", Matches, 1))
+
+	// test error presence
+	testCondError(t, newBoolCondition("banana", errorPresent, true))
+	testCondError(t, And(newBoolCondition("banana", errorPresent, true)))
+	testCondError(t, Or(newBoolCondition("banana", errorPresent, true)))
+	testCondError(t, newExistsCondition("banana", errorPresent))
+	testCondError(t, newFloatCondition("banana", errorPresent, 1.1))
+	testCondError(t, newIntCondition("banana", errorPresent, 1))
+	testCondError(t, newStringCondition("banana", errorPresent, "coconut"))
+	testCondError(t, newRegexCondition("banana", errorPresent, "coconut"))
+}
+
+func TestWhere(t *testing.T) {
+	c := Where("", 254, nil)
+	err := c.check()
+	if err == nil {
+		t.Error("should fail")
+	}
+}
diff --git a/database/query/operators.go b/database/query/operators.go
new file mode 100644
index 0000000..bbd21ee
--- /dev/null
+++ b/database/query/operators.go
@@ -0,0 +1,53 @@
+package query
+
+var (
+	operatorNames = map[string]uint8{
+		"==":         Equals,
+		">":          GreaterThan,
+		">=":         GreaterThanOrEqual,
+		"<":          LessThan,
+		"<=":         LessThanOrEqual,
+		"f==":        FloatEquals,
+		"f>":         FloatGreaterThan,
+		"f>=":        FloatGreaterThanOrEqual,
+		"f<":         FloatLessThan,
+		"f<=":        FloatLessThanOrEqual,
+		"sameas":     SameAs,
+		"s==":        SameAs,
+		"contains":   Contains,
+		"co":         Contains,
+		"startswith": StartsWith,
+		"sw":         StartsWith,
+		"endswith":   EndsWith,
+		"ew":         EndsWith,
+		"in":         In,
+		"matches":    Matches,
+		"re":         Matches,
+		"is":         Is,
+		"exists":     Exists,
+		"ex":         Exists,
+	}
+
+	primaryNames = make(map[uint8]string)
+)
+
+func init() {
+	for opName, opID := range operatorNames {
+		name, ok := primaryNames[opID]
+		if ok {
+			if len(name) < len(opName) {
+				primaryNames[opID] = opName
+			}
+		} else {
+			primaryNames[opID] = opName
+		}
+	}
+}
+
+func getOpName(operator uint8) string {
+	name, ok := primaryNames[operator]
+	if ok {
+		return name
+	}
+	return "[unknown]"
+}
diff --git a/database/query/operators_test.go b/database/query/operators_test.go
new file mode 100644
index 0000000..3f4fe81
--- /dev/null
+++ b/database/query/operators_test.go
@@ -0,0 +1,9 @@
+package query
+
+import "testing"
+
+func TestGetOpName(t *testing.T) {
+	if getOpName(254) != "[unknown]" {
+		t.Error("unexpected output")
+	}
+}
diff --git a/database/query/parser.go b/database/query/parser.go
index 2ceebd9..d14b847 100644
--- a/database/query/parser.go
+++ b/database/query/parser.go
@@ -4,77 +4,122 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
 )
 
-var (
-	operatorNames = map[string]uint8{
-		"==":         Equals,
-		">":          GreaterThan,
-		">=":         GreaterThanOrEqual,
-		"<":          LessThan,
-		"<=":         LessThanOrEqual,
-		"f==":        FloatEquals,
-		"f>":         FloatGreaterThan,
-		"f>=":        FloatGreaterThanOrEqual,
-		"f<":         FloatLessThan,
-		"f<=":        FloatLessThanOrEqual,
-		"sameas":     SameAs,
-		"s==":        SameAs,
-		"contains":   Contains,
-		"co":         Contains,
-		"startswith": StartsWith,
-		"sw":         StartsWith,
-		"endswith":   EndsWith,
-		"ew":         EndsWith,
-		"in":         In,
-		"matches":    Matches,
-		"re":         Matches,
-		"is":         Is,
-		"exists":     Exists,
-		"ex":         Exists,
-	}
-
-	primaryNames = make(map[uint8]string)
-)
-
-func init() {
-	for opName, opID := range operatorNames {
-		name, ok := primaryNames[opID]
-		if ok {
-			if len(name) < len(opName) {
-				primaryNames[opID] = opName
-			}
-		} else {
-			primaryNames[opID] = opName
-		}
-	}
-}
-
-func getOpName(operator uint8) string {
-	name, ok := primaryNames[operator]
-	if ok {
-		return name
-	}
-	return "[unknown]"
-}
-
-type treeElement struct {
-	branches       []*treeElement
+type snippet struct {
 	text           string
 	globalPosition int
 }
 
-var (
-	escapeReplacer = regexp.MustCompile("\\\\([^\\\\])")
-)
+// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
+func ParseQuery(query string) (*Query, error) {
+	snippets, err := extractSnippets(query)
+	if err != nil {
+		return nil, err
+	}
+	snippetsPos := 0
 
-// prepToken removes surrounding parenthesis and escape characters.
-func prepToken(text string) string {
-	return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
+	getSnippet := func() (*snippet, error) {
+		// order is important, as parseAndOr will always consume one additional snippet.
+		snippetsPos++
+		if snippetsPos > len(snippets) {
+			return nil, fmt.Errorf("unexpected end at position %d", len(query))
+		}
+		return snippets[snippetsPos-1], nil
+	}
+	remainingSnippets := func() int {
+		return len(snippets) - snippetsPos
+	}
+
+	// check for query word
+	queryWord, err := getSnippet()
+	if err != nil {
+		return nil, err
+	}
+	if queryWord.text != "query" {
+		return nil, errors.New("queries must start with \"query\"")
+	}
+
+	// get prefix
+	prefix, err := getSnippet()
+	if err != nil {
+		return nil, err
+	}
+	q := New(prefix.text)
+
+	for remainingSnippets() > 0 {
+		command, err := getSnippet()
+		if err != nil {
+			return nil, err
+		}
+
+		switch command.text {
+		case "where":
+			if q.where != nil {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			// parse conditions
+			condition, err := parseAndOr(getSnippet, remainingSnippets, true)
+			if err != nil {
+				return nil, err
+			}
+			// go one back, as parseAndOr had to check if its done
+			snippetsPos--
+
+			q.Where(condition)
+		case "orderby":
+			if q.orderBy != "" {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			orderBySnippet, err := getSnippet()
+			if err != nil {
+				return nil, err
+			}
+
+			q.OrderBy(orderBySnippet.text)
+		case "limit":
+			if q.limit != 0 {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			limitSnippet, err := getSnippet()
+			if err != nil {
+				return nil, err
+			}
+			limit, err := strconv.ParseUint(limitSnippet.text, 10, 31)
+			if err != nil {
+				return nil, fmt.Errorf("could not parse integer (%s) at position %d", limitSnippet.text, limitSnippet.globalPosition)
+			}
+
+			q.Limit(int(limit))
+		case "offset":
+			if q.offset != 0 {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			offsetSnippet, err := getSnippet()
+			if err != nil {
+				return nil, err
+			}
+			offset, err := strconv.ParseUint(offsetSnippet.text, 10, 31)
+			if err != nil {
+				return nil, fmt.Errorf("could not parse integer (%s) at position %d", offsetSnippet.text, offsetSnippet.globalPosition)
+			}
+
+			q.Offset(int(offset))
+		default:
+			return nil, fmt.Errorf("unknown clause \"%s\" at position %d", command.text, command.globalPosition)
+		}
+	}
+
+	return q.Check()
 }
 
-func extractSnippets(text string) (snippets []*treeElement, err error) {
+func extractSnippets(text string) (snippets []*snippet, err error) {
 
 	skip := false
 	start := -1
@@ -93,10 +138,10 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 			skip = true
 		}
 
-		// wait for parenthesis to be over
+		// wait for parenthesis to be overs
 		if inParenthesis {
 			if char == '"' {
-				snippets = append(snippets, &treeElement{
+				snippets = append(snippets, &snippet{
 					text:           prepToken(text[start+1 : pos]),
 					globalPosition: start + 1,
 				})
@@ -110,9 +155,9 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 		switch char {
 		case '\t', '\n', '\r', ' ', '(', ')':
 			if start >= 0 {
-				snippets = append(snippets, &treeElement{
+				snippets = append(snippets, &snippet{
 					text:           prepToken(text[start:pos]),
-					globalPosition: start,
+					globalPosition: start + 1,
 				})
 				start = -1
 			}
@@ -125,13 +170,13 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 		// handle special segment characters
 		switch char {
 		case '(', ')':
-			snippets = append(snippets, &treeElement{
+			snippets = append(snippets, &snippet{
 				text:           text[pos : pos+1],
-				globalPosition: pos,
+				globalPosition: pos + 1,
 			})
 		case '"':
 			if start < pos {
-				return nil, fmt.Errorf("parenthesis ('\"') may not be within words, please escape with '\\' (position: %d)", pos+1)
+				return nil, fmt.Errorf("parenthesis ('\"') may not be used within words, please escape with '\\' (position: %d)", pos+1)
 			}
 			inParenthesis = true
 		}
@@ -139,89 +184,166 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 	}
 
 	// add last
-	snippets = append(snippets, &treeElement{
-		text:           prepToken(text[start : pos+1]),
-		globalPosition: start,
-	})
+	if start >= 0 {
+		snippets = append(snippets, &snippet{
+			text:           prepToken(text[start : pos+1]),
+			globalPosition: start + 1,
+		})
+	}
 
 	return snippets, nil
 
 }
 
-// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
-func ParseQuery(query string) (*Query, error) {
-	snippets, err := extractSnippets(query)
-	if err != nil {
-		return nil, err
-	}
-	snippetsPos := 0
+func parseAndOr(getSnippet func() (*snippet, error), remainingSnippets func() int, rootCondition bool) (Condition, error) {
+	var isOr = false
+	var typeSet = false
+	var wrapInNot = false
+	var expectingMore = true
+	var conditions []Condition
 
-	getElement := func() (*treeElement, error) {
-		if snippetsPos >= len(snippets) {
-			return nil, fmt.Errorf("unexpected end at position %d", len(query))
+	for {
+		if !expectingMore && rootCondition && remainingSnippets() == 0 {
+			// advance snippetsPos by one, as it will be set back by 1
+			getSnippet()
+			if len(conditions) == 1 {
+				return conditions[0], nil
+			}
+			if isOr {
+				return Or(conditions...), nil
+			}
+			return And(conditions...), nil
 		}
-		return snippets[snippetsPos], nil
-	}
 
-	// check for query word
-	queryWord, err := getElement()
-	if err != nil {
-		return nil, err
-	}
-	if queryWord.text != "query" {
-		return nil, errors.New("queries must start with \"query\"")
-	}
+		firstSnippet, err := getSnippet()
+		if err != nil {
+			return nil, err
+		}
 
-	// get prefix
-	prefix, err := getElement()
-	if err != nil {
-		return nil, err
-	}
+		if !expectingMore && rootCondition {
+			switch firstSnippet.text {
+			case "orderby", "limit", "offset":
+				if len(conditions) == 1 {
+					return conditions[0], nil
+				}
+				if isOr {
+					return Or(conditions...), nil
+				}
+				return And(conditions...), nil
+			}
+		}
 
-	// check if no condition
-	if len(snippets) == 2 {
-		return New(prefix.text, nil)
+		switch firstSnippet.text {
+		case "(":
+			condition, err := parseAndOr(getSnippet, remainingSnippets, false)
+			if err != nil {
+				return nil, err
+			}
+			if wrapInNot {
+				conditions = append(conditions, Not(condition))
+				wrapInNot = false
+			} else {
+				conditions = append(conditions, condition)
+			}
+			expectingMore = true
+		case ")":
+			if len(conditions) == 1 {
+				return conditions[0], nil
+			}
+			if isOr {
+				return Or(conditions...), nil
+			}
+			return And(conditions...), nil
+		case "and":
+			if typeSet && isOr {
+				return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
+			}
+			isOr = false
+			typeSet = true
+			expectingMore = true
+		case "or":
+			if typeSet && !isOr {
+				return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
+			}
+			isOr = true
+			typeSet = true
+			expectingMore = true
+		case "not":
+			wrapInNot = true
+			expectingMore = true
+		default:
+			condition, err := parseCondition(firstSnippet, getSnippet)
+			if err != nil {
+				return nil, err
+			}
+			if wrapInNot {
+				conditions = append(conditions, Not(condition))
+				wrapInNot = false
+			} else {
+				conditions = append(conditions, condition)
+			}
+			expectingMore = false
+		}
 	}
-
-	// check for where word
-	whereWord, err := getElement()
-	if err != nil {
-		return nil, err
-	}
-	if whereWord.text != "where" {
-		return nil, errors.New("filtering queries must start conditions with \"where\"")
-	}
-
-	// parse conditions
-	condition, err := parseCondition(getElement)
-	if err != nil {
-		return nil, err
-	}
-
-	// check for additional tokens
-	// token := s.Scan()
-	// if token != scanner.EOF {
-	// 	return nil, fmt.Errorf("unexpected additional tokens at position %d", s.Position)
-	// }
-
-	return New(prefix.text, condition)
 }
 
-func parseCondition(getElement func() (*treeElement, error)) (Condition, error) {
-	first, err := getElement()
+func parseCondition(firstSnippet *snippet, getSnippet func() (*snippet, error)) (Condition, error) {
+	wrapInNot := false
+
+	// get operator name
+	opName, err := getSnippet()
 	if err != nil {
 		return nil, err
 	}
-
-	switch first.text {
-	case "(":
-		return parseAndOr(getElement, true, nil, false)
-		// case ""
+	// negate?
+	if opName.text == "not" {
+		wrapInNot = true
+		opName, err = getSnippet()
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	return nil, nil
+	// get operator
+	operator, ok := operatorNames[opName.text]
+	if !ok {
+		return nil, fmt.Errorf("unknown operator at position %d", opName.globalPosition)
+	}
+
+	// don't need a value for "exists"
+	if operator == Exists {
+		if wrapInNot {
+			return Not(Where(firstSnippet.text, operator, nil)), nil
+		}
+		return Where(firstSnippet.text, operator, nil), nil
+	}
+
+	// get value
+	value, err := getSnippet()
+	if err != nil {
+		return nil, err
+	}
+	if wrapInNot {
+		return Not(Where(firstSnippet.text, operator, value.text)), nil
+	}
+	return Where(firstSnippet.text, operator, value.text), nil
 }
 
-func parseAndOr(getElement func() (*treeElement, error), expectBracket bool, preParsedCondition Condition, preParsedIsOr bool) (Condition, error) {
-	return nil, nil
+var (
+	escapeReplacer = regexp.MustCompile("\\\\([^\\\\])")
+)
+
+// prepToken removes surrounding parenthesis and escape characters.
+func prepToken(text string) string {
+	return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
+}
+
+// escapeString correctly escapes a snippet for printing
+func escapeString(token string) string {
+	// check if token contains characters that need to be escaped
+	if strings.ContainsAny(token, "()\"\\\t\r\n ") {
+		// put the token in parenthesis and only escape \ and "
+		return fmt.Sprintf("\"%s\"", strings.Replace(token, "\"", "\\\"", -1))
+	}
+	return token
 }
diff --git a/database/query/parser_test.go b/database/query/parser_test.go
index 53cbd31..e467100 100644
--- a/database/query/parser_test.go
+++ b/database/query/parser_test.go
@@ -1,18 +1,168 @@
 package query
 
 import (
+	"reflect"
 	"testing"
+
+	"github.com/davecgh/go-spew/spew"
 )
 
-func TestMakeTree(t *testing.T) {
-	text1 := `(age > 100 and experience <= "99") or (age < 10 and motivation > 50) or name matches Mr.\"X or name matches y`
+func TestExtractSnippets(t *testing.T) {
+	text1 := `query test: where ( "bananas" > 100 and monkeys.# <= "12")or(coconuts < 10 "and" area > 50) or name sameas Julian or name matches ^King\ `
+	result1 := []*snippet{
+		&snippet{text: "query", globalPosition: 1},
+		&snippet{text: "test:", globalPosition: 7},
+		&snippet{text: "where", globalPosition: 13},
+		&snippet{text: "(", globalPosition: 19},
+		&snippet{text: "bananas", globalPosition: 21},
+		&snippet{text: ">", globalPosition: 31},
+		&snippet{text: "100", globalPosition: 33},
+		&snippet{text: "and", globalPosition: 37},
+		&snippet{text: "monkeys.#", globalPosition: 41},
+		&snippet{text: "<=", globalPosition: 51},
+		&snippet{text: "12", globalPosition: 54},
+		&snippet{text: ")", globalPosition: 58},
+		&snippet{text: "or", globalPosition: 59},
+		&snippet{text: "(", globalPosition: 61},
+		&snippet{text: "coconuts", globalPosition: 62},
+		&snippet{text: "<", globalPosition: 71},
+		&snippet{text: "10", globalPosition: 73},
+		&snippet{text: "and", globalPosition: 76},
+		&snippet{text: "area", globalPosition: 82},
+		&snippet{text: ">", globalPosition: 87},
+		&snippet{text: "50", globalPosition: 89},
+		&snippet{text: ")", globalPosition: 91},
+		&snippet{text: "or", globalPosition: 93},
+		&snippet{text: "name", globalPosition: 96},
+		&snippet{text: "sameas", globalPosition: 101},
+		&snippet{text: "Julian", globalPosition: 108},
+		&snippet{text: "or", globalPosition: 115},
+		&snippet{text: "name", globalPosition: 118},
+		&snippet{text: "matches", globalPosition: 123},
+		&snippet{text: "^King ", globalPosition: 131},
+	}
+
 	snippets, err := extractSnippets(text1)
 	if err != nil {
-		t.Errorf("failed to make tree: %s", err)
-	} else {
+		t.Errorf("failed to extract snippets: %s", err)
+	}
+
+	if !reflect.DeepEqual(result1, snippets) {
+		t.Errorf("unexpected results:")
 		for _, el := range snippets {
 			t.Errorf("%+v", el)
 		}
 	}
+
 	// t.Error(spew.Sprintf("%v", treeElement))
 }
+
+func testParsing(t *testing.T, queryText string, expectedResult *Query) {
+	_, err := expectedResult.Check()
+	if err != nil {
+		t.Errorf("failed to create query: %s", err)
+		return
+	}
+
+	q, err := ParseQuery(queryText)
+	if err != nil {
+		t.Errorf("failed to parse query: %s", err)
+		return
+	}
+
+	if queryText != q.Print() {
+		t.Errorf("string match failed: %s", q.Print())
+		return
+	}
+	if !reflect.DeepEqual(expectedResult, q) {
+		t.Error("deepqual match failed.")
+		t.Error("got:")
+		t.Error(spew.Sdump(q))
+		t.Error("expected:")
+		t.Error(spew.Sdump(expectedResult))
+	}
+}
+
+func TestParseQuery(t *testing.T) {
+	text1 := `query test: where (bananas > 100 and monkeys.# <= 12) or not (coconuts < 10 and area not > 50) or name sameas Julian or name matches "^King " orderby name limit 10 offset 20`
+	result1 := New("test:").Where(Or(
+		And(
+			Where("bananas", GreaterThan, 100),
+			Where("monkeys.#", LessThanOrEqual, 12),
+		),
+		Not(And(
+			Where("coconuts", LessThan, 10),
+			Not(Where("area", GreaterThan, 50)),
+		)),
+		Where("name", SameAs, "Julian"),
+		Where("name", Matches, "^King "),
+	)).OrderBy("name").Limit(10).Offset(20)
+	testParsing(t, text1, result1)
+
+	testParsing(t, `query test: orderby name`, New("test:").OrderBy("name"))
+	testParsing(t, `query test: limit 10`, New("test:").Limit(10))
+	testParsing(t, `query test: offset 10`, New("test:").Offset(10))
+	testParsing(t, `query test: where banana matches ^ban`, New("test:").Where(Where("banana", Matches, "^ban")))
+	testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
+	testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
+
+	// test all operators
+	testParsing(t, `query test: where banana == 1`, New("test:").Where(Where("banana", Equals, 1)))
+	testParsing(t, `query test: where banana > 1`, New("test:").Where(Where("banana", GreaterThan, 1)))
+	testParsing(t, `query test: where banana >= 1`, New("test:").Where(Where("banana", GreaterThanOrEqual, 1)))
+	testParsing(t, `query test: where banana < 1`, New("test:").Where(Where("banana", LessThan, 1)))
+	testParsing(t, `query test: where banana <= 1`, New("test:").Where(Where("banana", LessThanOrEqual, 1)))
+	testParsing(t, `query test: where banana f== 1.1`, New("test:").Where(Where("banana", FloatEquals, 1.1)))
+	testParsing(t, `query test: where banana f> 1.1`, New("test:").Where(Where("banana", FloatGreaterThan, 1.1)))
+	testParsing(t, `query test: where banana f>= 1.1`, New("test:").Where(Where("banana", FloatGreaterThanOrEqual, 1.1)))
+	testParsing(t, `query test: where banana f< 1.1`, New("test:").Where(Where("banana", FloatLessThan, 1.1)))
+	testParsing(t, `query test: where banana f<= 1.1`, New("test:").Where(Where("banana", FloatLessThanOrEqual, 1.1)))
+	testParsing(t, `query test: where banana sameas banana`, New("test:").Where(Where("banana", SameAs, "banana")))
+	testParsing(t, `query test: where banana contains banana`, New("test:").Where(Where("banana", Contains, "banana")))
+	testParsing(t, `query test: where banana startswith banana`, New("test:").Where(Where("banana", StartsWith, "banana")))
+	testParsing(t, `query test: where banana endswith banana`, New("test:").Where(Where("banana", EndsWith, "banana")))
+	testParsing(t, `query test: where banana in banana,coconut`, New("test:").Where(Where("banana", In, []string{"banana", "coconut"})))
+	testParsing(t, `query test: where banana matches banana`, New("test:").Where(Where("banana", Matches, "banana")))
+	testParsing(t, `query test: where banana is true`, New("test:").Where(Where("banana", Is, true)))
+	testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
+
+	// special
+	testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
+}
+
+func testParseError(t *testing.T, queryText string, expectedErrorString string) {
+	_, err := ParseQuery(queryText)
+	if err == nil {
+		t.Errorf("should fail to parse: %s", queryText)
+		return
+	}
+	if err.Error() != expectedErrorString {
+		t.Errorf("unexpected error for query: %s\nwanted: %s\n   got: %s", queryText, expectedErrorString, err)
+	}
+}
+
+func TestParseErrors(t *testing.T) {
+	// syntax
+	testParseError(t, `query`, `unexpected end at position 5`)
+	testParseError(t, `query test`, `invalid prefix: test`)
+	testParseError(t, `query test: where`, `unexpected end at position 17`)
+	testParseError(t, `query test: where (`, `unexpected end at position 19`)
+	testParseError(t, `query test: where )`, `unknown clause ")" at position 19`)
+	testParseError(t, `query test: where not`, `unexpected end at position 21`)
+	testParseError(t, `query test: where banana`, `unexpected end at position 24`)
+	testParseError(t, `query test: where banana >`, `unexpected end at position 26`)
+	testParseError(t, `query test: where banana nope`, `unknown operator at position 26`)
+	testParseError(t, `query test: where banana exists or`, `unexpected end at position 34`)
+	testParseError(t, `query test: where banana exists and`, `unexpected end at position 35`)
+	testParseError(t, `query test: where banana exists and (`, `unexpected end at position 37`)
+	testParseError(t, `query test: where banana exists and banana is true or`, `you may not mix "and" and "or" (position: 52)`)
+	testParseError(t, `query test: where banana exists or banana is true and`, `you may not mix "and" and "or" (position: 51)`)
+	// testParseError(t, `query test: where banana exists and (`, ``)
+
+	// value parsing error
+	testParseError(t, `query test: where banana == banana`, `could not parse banana to int64: strconv.ParseInt: parsing "banana": invalid syntax (hint: use "sameas" to compare strings)`)
+	testParseError(t, `query test: where banana f== banana`, `could not parse banana to float64: strconv.ParseFloat: parsing "banana": invalid syntax`)
+	testParseError(t, `query test: where banana in banana`, `could not parse "banana" to []string`)
+	testParseError(t, `query test: where banana matches [banana`, "could not compile regex \"[banana\": error parsing regexp: missing closing ]: `[banana`")
+	testParseError(t, `query test: where banana is great`, `could not parse "great" to bool: strconv.ParseBool: parsing "great": invalid syntax`)
+}
diff --git a/database/query/query.go b/database/query/query.go
index c8fbf1e..8b4433e 100644
--- a/database/query/query.go
+++ b/database/query/query.go
@@ -22,56 +22,113 @@ var (
 
 // Query contains a compiled query.
 type Query struct {
-	prefix    string
-	condition Condition
+	checked bool
+	prefix  string
+	where   Condition
+	orderBy string
+	limit   int
+	offset  int
 }
 
-// New creates a new query.
-func New(prefix string, condition Condition) (*Query, error) {
+// New creates a new query with the supplied prefix.
+func New(prefix string) *Query {
+	return &Query{
+		prefix: prefix,
+	}
+}
+
+// Where adds filtering.
+func (q *Query) Where(condition Condition) *Query {
+	q.where = condition
+	return q
+}
+
+// Limit limits the number of returned results.
+func (q *Query) Limit(limit int) *Query {
+	q.limit = limit
+	return q
+}
+
+// Offset sets the query offset.
+func (q *Query) Offset(offset int) *Query {
+	q.offset = offset
+	return q
+}
+
+// OrderBy orders the results by the given key.
+func (q *Query) OrderBy(key string) *Query {
+	q.orderBy = key
+	return q
+}
+
+// Check checks for errors in the query.
+func (q *Query) Check() (*Query, error) {
+	if q.checked {
+		return q, nil
+	}
+
 	// check prefix
-	if !prefixExpr.MatchString(prefix) {
-		return nil, fmt.Errorf("invalid prefix: %s", prefix)
+	if !prefixExpr.MatchString(q.prefix) {
+		return nil, fmt.Errorf("invalid prefix: %s", q.prefix)
 	}
 
 	// check condition
-	if condition != nil {
-		err := condition.check()
+	if q.where != nil {
+		err := q.where.check()
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		condition = &noCond{}
+		q.where = &noCond{}
 	}
 
-	// return query
-	return &Query{
-		prefix:    prefix,
-		condition: condition,
-	}, nil
+	q.checked = true
+	return q, nil
 }
 
-// MustCompile creates a new query and panics on an error.
-func MustCompile(prefix string, condition Condition) *Query {
-	q, err := New(prefix, condition)
+// MustBeValid checks for errors in the query and panics if there is an error.
+func (q *Query) MustBeValid() *Query {
+	_, err := q.Check()
 	if err != nil {
 		panic(err)
 	}
 	return q
 }
 
-// Matches checks whether the query matches the supplied data object.
-func (q *Query) Matches(f Fetcher) bool {
-	return q.condition.complies(f)
+// IsChecked returns whether they query was checked.
+func (q *Query) IsChecked() bool {
+	return q.checked
 }
 
-// String returns the string representation of the query.
-func (q *Query) String() string {
-	text := q.condition.string()
-	if text == "" {
-		return fmt.Sprintf("query %s", q.prefix)
-	}
-	if strings.HasPrefix(text, "(") {
-		text = text[1 : len(text)-1]
-	}
-	return fmt.Sprintf("query %s where %s", q.prefix, text)
+// Matches checks whether the query matches the supplied data object.
+func (q *Query) Matches(f Fetcher) bool {
+	return q.where.complies(f)
+}
+
+// Print returns the string representation of the query.
+func (q *Query) Print() string {
+	where := q.where.string()
+	if where != "" {
+		if strings.HasPrefix(where, "(") {
+			where = where[1 : len(where)-1]
+		}
+		where = fmt.Sprintf(" where %s", where)
+	}
+
+	var orderBy string
+	if q.orderBy != "" {
+		orderBy = fmt.Sprintf(" orderby %s", q.orderBy)
+	}
+
+	var limit string
+	if q.limit > 0 {
+		limit = fmt.Sprintf(" limit %d", q.limit)
+	}
+
+	var offset string
+	if q.offset > 0 {
+		offset = fmt.Sprintf(" offset %d", q.offset)
+	}
+
+	return fmt.Sprintf("query %s%s%s%s%s", q.prefix, where, orderBy, limit, offset)
 }
diff --git a/database/query/query_test.go b/database/query/query_test.go
index 39ce970..bf25e70 100644
--- a/database/query/query_test.go
+++ b/database/query/query_test.go
@@ -6,7 +6,7 @@ import (
 
 var (
 	// copied from https://github.com/tidwall/gjson/blob/master/gjson_test.go
-	testJson = `{"age":100, "name":{"here":"B\\\"R"},
+	testJSON = `{"age":100, "name":{"here":"B\\\"R"},
   "noop":{"what is a wren?":"a bird"},
   "happy":true,"immortal":false,
   "items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7],
@@ -45,25 +45,25 @@ var (
 )
 
 func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) {
-	q := MustCompile("test:", condition)
+	q := New("test:").Where(condition).MustBeValid()
 
 	// fmt.Printf("%s\n", q.String())
 
 	matched := q.Matches(f)
 	switch {
 	case !matched && shouldMatch:
-		t.Errorf("should match: %s", q.String())
+		t.Errorf("should match: %s", q.Print())
 	case matched && !shouldMatch:
-		t.Errorf("should not match: %s", q.String())
+		t.Errorf("should not match: %s", q.Print())
 	}
 }
 
 func TestQuery(t *testing.T) {
 
-	// if !gjson.Valid(testJson) {
+	// if !gjson.Valid(testJSON) {
 	// 	t.Fatal("test json is invalid")
 	// }
-	f := NewJSONFetcher(testJson)
+	f := NewJSONFetcher(testJSON)
 
 	testQuery(t, f, true, Where("age", Equals, 100))
 	testQuery(t, f, true, Where("age", GreaterThan, uint8(99)))
@@ -88,6 +88,7 @@ func TestQuery(t *testing.T) {
 	testQuery(t, f, true, Where("lastly.yay", In, "draft,final"))
 	testQuery(t, f, true, Where("lastly.yay", In, "final,draft"))
 
+	testQuery(t, f, true, Where("happy", Is, true))
 	testQuery(t, f, true, Where("happy", Is, "true"))
 	testQuery(t, f, true, Where("happy", Is, "t"))
 	testQuery(t, f, true, Not(Where("happy", Is, "0")))

From f8ff7d143e25956e1d9d57be4f7187085fe46d41 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 31 Aug 2018 17:36:32 +0200
Subject: [PATCH 04/20] Start database interface/model/iterator revamp

---
 database/iterator/iterator.go     | 18 ++++++++++++++++++
 database/model/model.go           |  7 +++++++
 database/storage/badger/badger.go |  1 +
 database/storage/bbolt/bbolt.go   |  9 +++++++++
 database/storage/interface.go     | 30 ++++++++++++++++++++++++++++++
 database/utils/kvops/kvops.go     |  1 +
 6 files changed, 66 insertions(+)
 create mode 100644 database/iterator/iterator.go
 create mode 100644 database/model/model.go
 create mode 100644 database/storage/badger/badger.go
 create mode 100644 database/storage/bbolt/bbolt.go
 create mode 100644 database/storage/interface.go
 create mode 100644 database/utils/kvops/kvops.go

diff --git a/database/iterator/iterator.go b/database/iterator/iterator.go
new file mode 100644
index 0000000..e045d6b
--- /dev/null
+++ b/database/iterator/iterator.go
@@ -0,0 +1,18 @@
+package iterator
+
+import (
+	"github.com/Safing/portbase/database/model"
+)
+
+// Iterator defines the iterator structure.
+type Iterator struct {
+	Next  chan model.Model
+	Error error
+}
+
+// New creates a new Iterator.
+func New() *Iterator {
+	return &Iterator{
+		Next: make(chan model.Model, 10),
+	}
+}
diff --git a/database/model/model.go b/database/model/model.go
new file mode 100644
index 0000000..7aa1211
--- /dev/null
+++ b/database/model/model.go
@@ -0,0 +1,7 @@
+package model
+
+type Meta struct {
+}
+
+type Model interface {
+}
diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
new file mode 100644
index 0000000..2b90f77
--- /dev/null
+++ b/database/storage/badger/badger.go
@@ -0,0 +1 @@
+package badger
diff --git a/database/storage/bbolt/bbolt.go b/database/storage/bbolt/bbolt.go
new file mode 100644
index 0000000..0c18c51
--- /dev/null
+++ b/database/storage/bbolt/bbolt.go
@@ -0,0 +1,9 @@
+package bbolt
+
+import bolt "go.etcd.io/bbolt"
+
+db, err := bolt.Open(path, 0666, nil)
+if err != nil {
+  return err
+}
+defer db.Close()
diff --git a/database/storage/interface.go b/database/storage/interface.go
new file mode 100644
index 0000000..108a25f
--- /dev/null
+++ b/database/storage/interface.go
@@ -0,0 +1,30 @@
+package storage
+
+import (
+  "github.com/Safing/portbase/database/iterator"
+  "github.com/Safing/portbase/database/model"
+)
+
+// Interface defines the database storage API.
+type Interface interface {
+  // Full
+  Exists(key string) (bool, error)
+  Get(key string) (model.Model, error)
+  Create(key string, model model.Model) error
+  Update(key string, model model.Model) error // create when not exists
+  UpdateOrCreate(key string, model model.Model) error // update, create if not exists.
+  Delete(key string) error
+
+  // Partial
+  // What happens if I mutate a value that does not yet exist? How would I know its type?
+  InsertPartial(key string, partialObject interface{})
+  InsertValue(key string, attribute string, value interface{})
+
+  // Query
+  Query(*query.Query) (*iterator.Iterator, error)
+
+  // Meta
+  LetExpire(key string, timestamp int64) error
+  MakeSecret(key string) error // only visible internal
+  MakeCrownJewel(key string) error // do not sync between devices
+}
diff --git a/database/utils/kvops/kvops.go b/database/utils/kvops/kvops.go
new file mode 100644
index 0000000..bf8ee70
--- /dev/null
+++ b/database/utils/kvops/kvops.go
@@ -0,0 +1 @@
+package kvops

From 307ddd70fd57edd2e63eef64e04aa568f3af6280 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Mon, 3 Sep 2018 17:07:23 +0200
Subject: [PATCH 05/20] Add benchmarks for un/serializing model Meta

---
 database/model/meta.colf           |  10 +
 database/model/meta.gencode        |   8 +
 database/model/meta.gencode.gen.go | 163 ++++++++++
 database/model/meta.go             |  10 +
 database/model/meta_test.go        | 311 ++++++++++++++++++
 database/model/model.go            |   3 -
 database/model/model/Colfer.go     | 507 +++++++++++++++++++++++++++++
 7 files changed, 1009 insertions(+), 3 deletions(-)
 create mode 100644 database/model/meta.colf
 create mode 100644 database/model/meta.gencode
 create mode 100644 database/model/meta.gencode.gen.go
 create mode 100644 database/model/meta.go
 create mode 100644 database/model/meta_test.go
 create mode 100644 database/model/model/Colfer.go

diff --git a/database/model/meta.colf b/database/model/meta.colf
new file mode 100644
index 0000000..7cc99f4
--- /dev/null
+++ b/database/model/meta.colf
@@ -0,0 +1,10 @@
+package model
+
+type course struct {
+  Created   int64
+  Modified  int64
+  Expires   int64
+  Deleted   int64
+  Secret    bool
+  Cronjewel bool
+}
diff --git a/database/model/meta.gencode b/database/model/meta.gencode
new file mode 100644
index 0000000..7592e2d
--- /dev/null
+++ b/database/model/meta.gencode
@@ -0,0 +1,8 @@
+struct Meta {
+	Created   int64
+	Modified  int64
+	Expires   int64
+	Deleted   int64
+	Secret    bool
+	Cronjewel bool
+}
diff --git a/database/model/meta.gencode.gen.go b/database/model/meta.gencode.gen.go
new file mode 100644
index 0000000..4cf07f0
--- /dev/null
+++ b/database/model/meta.gencode.gen.go
@@ -0,0 +1,163 @@
+package model
+
+import (
+	"io"
+	"time"
+	"unsafe"
+)
+
+var (
+	_ = unsafe.Sizeof(0)
+	_ = io.ReadFull
+	_ = time.Now()
+)
+
+// type Meta struct {
+// 	Created   int64
+// 	Modified  int64
+// 	Expires   int64
+// 	Deleted   int64
+// 	Secret    bool
+// 	Cronjewel bool
+// }
+
+func (d *Meta) Size() (s uint64) {
+
+	s += 34
+	return
+}
+func (d *Meta) Marshal(buf []byte) ([]byte, error) {
+	size := d.Size()
+	{
+		if uint64(cap(buf)) >= size {
+			buf = buf[:size]
+		} else {
+			buf = make([]byte, size)
+		}
+	}
+	i := uint64(0)
+
+	{
+
+		buf[0+0] = byte(d.Created >> 0)
+
+		buf[1+0] = byte(d.Created >> 8)
+
+		buf[2+0] = byte(d.Created >> 16)
+
+		buf[3+0] = byte(d.Created >> 24)
+
+		buf[4+0] = byte(d.Created >> 32)
+
+		buf[5+0] = byte(d.Created >> 40)
+
+		buf[6+0] = byte(d.Created >> 48)
+
+		buf[7+0] = byte(d.Created >> 56)
+
+	}
+	{
+
+		buf[0+8] = byte(d.Modified >> 0)
+
+		buf[1+8] = byte(d.Modified >> 8)
+
+		buf[2+8] = byte(d.Modified >> 16)
+
+		buf[3+8] = byte(d.Modified >> 24)
+
+		buf[4+8] = byte(d.Modified >> 32)
+
+		buf[5+8] = byte(d.Modified >> 40)
+
+		buf[6+8] = byte(d.Modified >> 48)
+
+		buf[7+8] = byte(d.Modified >> 56)
+
+	}
+	{
+
+		buf[0+16] = byte(d.Expires >> 0)
+
+		buf[1+16] = byte(d.Expires >> 8)
+
+		buf[2+16] = byte(d.Expires >> 16)
+
+		buf[3+16] = byte(d.Expires >> 24)
+
+		buf[4+16] = byte(d.Expires >> 32)
+
+		buf[5+16] = byte(d.Expires >> 40)
+
+		buf[6+16] = byte(d.Expires >> 48)
+
+		buf[7+16] = byte(d.Expires >> 56)
+
+	}
+	{
+
+		buf[0+24] = byte(d.Deleted >> 0)
+
+		buf[1+24] = byte(d.Deleted >> 8)
+
+		buf[2+24] = byte(d.Deleted >> 16)
+
+		buf[3+24] = byte(d.Deleted >> 24)
+
+		buf[4+24] = byte(d.Deleted >> 32)
+
+		buf[5+24] = byte(d.Deleted >> 40)
+
+		buf[6+24] = byte(d.Deleted >> 48)
+
+		buf[7+24] = byte(d.Deleted >> 56)
+
+	}
+	{
+		if d.Secret {
+			buf[32] = 1
+		} else {
+			buf[32] = 0
+		}
+	}
+	{
+		if d.Cronjewel {
+			buf[33] = 1
+		} else {
+			buf[33] = 0
+		}
+	}
+	return buf[:i+34], nil
+}
+
+func (d *Meta) Unmarshal(buf []byte) (uint64, error) {
+	i := uint64(0)
+
+	{
+
+		d.Created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
+
+	}
+	{
+
+		d.Modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
+
+	}
+	{
+
+		d.Expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
+
+	}
+	{
+
+		d.Deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
+
+	}
+	{
+		d.Secret = buf[32] == 1
+	}
+	{
+		d.Cronjewel = buf[33] == 1
+	}
+	return i + 34, nil
+}
diff --git a/database/model/meta.go b/database/model/meta.go
new file mode 100644
index 0000000..161ee64
--- /dev/null
+++ b/database/model/meta.go
@@ -0,0 +1,10 @@
+package model
+
+type Meta struct {
+	Created   int64 `json:"c,omitempty" bson:"c,omitempty"`
+	Modified  int64 `json:"m,omitempty" bson:"m,omitempty"`
+	Expires   int64 `json:"e,omitempty" bson:"e,omitempty"`
+	Deleted   int64 `json:"d,omitempty" bson:"d,omitempty"`
+	Secret    bool  `json:"s,omitempty" bson:"s,omitempty"` // secrets must not be sent to the UI, only synced between nodes
+	Cronjewel bool  `json:"j,omitempty" bson:"j,omitempty"` // crownjewels must never leave the instance, but may be read by the UI
+}
diff --git a/database/model/meta_test.go b/database/model/meta_test.go
new file mode 100644
index 0000000..4141fce
--- /dev/null
+++ b/database/model/meta_test.go
@@ -0,0 +1,311 @@
+package model
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/Safing/portbase/container"
+	"github.com/Safing/portbase/database/model/model"
+	"github.com/Safing/portbase/formats/dsd"
+	xdr2 "github.com/davecgh/go-xdr/xdr2"
+)
+
+var (
+	testMeta = &Meta{
+		Created:   time.Now().Unix(),
+		Modified:  time.Now().Unix(),
+		Expires:   time.Now().Unix(),
+		Deleted:   time.Now().Unix(),
+		Secret:    true,
+		Cronjewel: true,
+	}
+)
+
+func BenchmarkAllocateBytes(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = make([]byte, 33)
+	}
+}
+
+func BenchmarkAllocateStruct1(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var new Meta
+		_ = new
+	}
+}
+
+func BenchmarkAllocateStruct2(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = Meta{}
+	}
+}
+
+func BenchmarkMetaSerializeCustom(b *testing.B) {
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		c := container.New()
+		c.AppendNumber(uint64(testMeta.Created))
+		c.AppendNumber(uint64(testMeta.Modified))
+		c.AppendNumber(uint64(testMeta.Expires))
+		c.AppendNumber(uint64(testMeta.Deleted))
+		switch {
+		case testMeta.Secret && testMeta.Cronjewel:
+			c.AppendNumber(3)
+		case testMeta.Secret:
+			c.AppendNumber(1)
+		case testMeta.Cronjewel:
+			c.AppendNumber(2)
+		default:
+			c.AppendNumber(0)
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeCustom(b *testing.B) {
+
+	// Setup
+	c := container.New()
+	c.AppendNumber(uint64(testMeta.Created))
+	c.AppendNumber(uint64(testMeta.Modified))
+	c.AppendNumber(uint64(testMeta.Expires))
+	c.AppendNumber(uint64(testMeta.Deleted))
+	switch {
+	case testMeta.Secret && testMeta.Cronjewel:
+		c.AppendNumber(3)
+	case testMeta.Secret:
+		c.AppendNumber(1)
+	case testMeta.Cronjewel:
+		c.AppendNumber(2)
+	default:
+		c.AppendNumber(0)
+	}
+	encodedData := c.CompileData()
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		var err error
+		var num uint64
+		c := container.New(encodedData)
+		num, err = c.GetNextN64()
+		newMeta.Created = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.Modified = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.Expires = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.Deleted = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+
+		flags, err := c.GetNextN8()
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+
+		switch flags {
+		case 3:
+			newMeta.Secret = true
+			newMeta.Cronjewel = true
+		case 2:
+			newMeta.Cronjewel = true
+		case 1:
+			newMeta.Secret = true
+		case 0:
+		default:
+			b.Errorf("invalid flag value: %d", flags)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaSerializeWithXDR2(b *testing.B) {
+
+	// Setup
+	var w bytes.Buffer
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		w.Reset()
+		_, err := xdr2.Marshal(&w, testMeta)
+		if err != nil {
+			b.Errorf("failed to serialize with xdr2: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeWithXDR2(b *testing.B) {
+
+	// Setup
+	var w bytes.Buffer
+	_, err := xdr2.Marshal(&w, testMeta)
+	if err != nil {
+		b.Errorf("failed to serialize with xdr2: %s", err)
+	}
+	encodedData := w.Bytes()
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		_, err := xdr2.Unmarshal(bytes.NewReader(encodedData), &newMeta)
+		if err != nil {
+			b.Errorf("failed to unserialize with xdr2: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaSerializeWithColfer(b *testing.B) {
+
+	testColf := &model.Course{
+		Created:   time.Now().Unix(),
+		Modified:  time.Now().Unix(),
+		Expires:   time.Now().Unix(),
+		Deleted:   time.Now().Unix(),
+		Secret:    true,
+		Cronjewel: true,
+	}
+
+	// Setup
+	for i := 0; i < b.N; i++ {
+		_, err := testColf.MarshalBinary()
+		if err != nil {
+			b.Errorf("failed to serialize with colfer: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeWithColfer(b *testing.B) {
+
+	testColf := &model.Course{
+		Created:   time.Now().Unix(),
+		Modified:  time.Now().Unix(),
+		Expires:   time.Now().Unix(),
+		Deleted:   time.Now().Unix(),
+		Secret:    true,
+		Cronjewel: true,
+	}
+	encodedData, err := testColf.MarshalBinary()
+	if err != nil {
+		b.Errorf("failed to serialize with colfer: %s", err)
+		return
+	}
+
+	// Setup
+	for i := 0; i < b.N; i++ {
+		var testUnColf model.Course
+		err := testUnColf.UnmarshalBinary(encodedData)
+		if err != nil {
+			b.Errorf("failed to unserialize with colfer: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaSerializeWithCodegen(b *testing.B) {
+
+	for i := 0; i < b.N; i++ {
+		buf := make([]byte, 34)
+		_, err := testMeta.Marshal(buf)
+		if err != nil {
+			b.Errorf("failed to serialize with codegen: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
+
+	// Setup
+	buf := make([]byte, 34)
+	encodedData, err := testMeta.Marshal(buf)
+	if err != nil {
+		b.Errorf("failed to serialize with codegen: %s", err)
+		return
+	}
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		_, err := newMeta.Unmarshal(encodedData)
+		if err != nil {
+			b.Errorf("failed to unserialize with codegen: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
+
+	for i := 0; i < b.N; i++ {
+		_, err := dsd.Dump(testMeta, dsd.JSON)
+		if err != nil {
+			b.Errorf("failed to serialize with DSD/JSON: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
+
+	// Setup
+	encodedData, err := dsd.Dump(testMeta, dsd.JSON)
+	if err != nil {
+		b.Errorf("failed to serialize with DSD/JSON: %s", err)
+		return
+	}
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		_, err := dsd.Load(encodedData, &newMeta)
+		if err != nil {
+			b.Errorf("failed to unserialize with DSD/JSON: %s", err)
+			return
+		}
+	}
+
+}
diff --git a/database/model/model.go b/database/model/model.go
index 7aa1211..93debc9 100644
--- a/database/model/model.go
+++ b/database/model/model.go
@@ -1,7 +1,4 @@
 package model
 
-type Meta struct {
-}
-
 type Model interface {
 }
diff --git a/database/model/model/Colfer.go b/database/model/model/Colfer.go
new file mode 100644
index 0000000..0f68a19
--- /dev/null
+++ b/database/model/model/Colfer.go
@@ -0,0 +1,507 @@
+package model
+
+// Code generated by colf(1); DO NOT EDIT.
+// The compiler used schema file meta.colf.
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+)
+
+var intconv = binary.BigEndian
+
+// Colfer configuration attributes
+var (
+	// ColferSizeMax is the upper limit for serial byte sizes.
+	ColferSizeMax = 16 * 1024 * 1024
+)
+
+// ColferMax signals an upper limit breach.
+type ColferMax string
+
+// Error honors the error interface.
+func (m ColferMax) Error() string { return string(m) }
+
+// ColferError signals a data mismatch as as a byte index.
+type ColferError int
+
+// Error honors the error interface.
+func (i ColferError) Error() string {
+	return fmt.Sprintf("colfer: unknown header at byte %d", i)
+}
+
+// ColferTail signals data continuation as a byte index.
+type ColferTail int
+
+// Error honors the error interface.
+func (i ColferTail) Error() string {
+	return fmt.Sprintf("colfer: data continuation at byte %d", i)
+}
+
+type Course struct {
+	Created int64
+
+	Modified int64
+
+	Expires int64
+
+	Deleted int64
+
+	Secret bool
+
+	Cronjewel bool
+}
+
+// MarshalTo encodes o as Colfer into buf and returns the number of bytes written.
+// If the buffer is too small, MarshalTo will panic.
+func (o *Course) MarshalTo(buf []byte) int {
+	var i int
+
+	if v := o.Created; v != 0 {
+		x := uint64(v)
+		if v >= 0 {
+			buf[i] = 0
+		} else {
+			x = ^x + 1
+			buf[i] = 0 | 0x80
+		}
+		i++
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			buf[i] = byte(x | 0x80)
+			x >>= 7
+			i++
+		}
+		buf[i] = byte(x)
+		i++
+	}
+
+	if v := o.Modified; v != 0 {
+		x := uint64(v)
+		if v >= 0 {
+			buf[i] = 1
+		} else {
+			x = ^x + 1
+			buf[i] = 1 | 0x80
+		}
+		i++
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			buf[i] = byte(x | 0x80)
+			x >>= 7
+			i++
+		}
+		buf[i] = byte(x)
+		i++
+	}
+
+	if v := o.Expires; v != 0 {
+		x := uint64(v)
+		if v >= 0 {
+			buf[i] = 2
+		} else {
+			x = ^x + 1
+			buf[i] = 2 | 0x80
+		}
+		i++
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			buf[i] = byte(x | 0x80)
+			x >>= 7
+			i++
+		}
+		buf[i] = byte(x)
+		i++
+	}
+
+	if v := o.Deleted; v != 0 {
+		x := uint64(v)
+		if v >= 0 {
+			buf[i] = 3
+		} else {
+			x = ^x + 1
+			buf[i] = 3 | 0x80
+		}
+		i++
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			buf[i] = byte(x | 0x80)
+			x >>= 7
+			i++
+		}
+		buf[i] = byte(x)
+		i++
+	}
+
+	if o.Secret {
+		buf[i] = 4
+		i++
+	}
+
+	if o.Cronjewel {
+		buf[i] = 5
+		i++
+	}
+
+	buf[i] = 0x7f
+	i++
+	return i
+}
+
+// MarshalLen returns the Colfer serial byte size.
+// The error return option is model.ColferMax.
+func (o *Course) MarshalLen() (int, error) {
+	l := 1
+
+	if v := o.Created; v != 0 {
+		l += 2
+		x := uint64(v)
+		if v < 0 {
+			x = ^x + 1
+		}
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			x >>= 7
+			l++
+		}
+	}
+
+	if v := o.Modified; v != 0 {
+		l += 2
+		x := uint64(v)
+		if v < 0 {
+			x = ^x + 1
+		}
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			x >>= 7
+			l++
+		}
+	}
+
+	if v := o.Expires; v != 0 {
+		l += 2
+		x := uint64(v)
+		if v < 0 {
+			x = ^x + 1
+		}
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			x >>= 7
+			l++
+		}
+	}
+
+	if v := o.Deleted; v != 0 {
+		l += 2
+		x := uint64(v)
+		if v < 0 {
+			x = ^x + 1
+		}
+		for n := 0; x >= 0x80 && n < 8; n++ {
+			x >>= 7
+			l++
+		}
+	}
+
+	if o.Secret {
+		l++
+	}
+
+	if o.Cronjewel {
+		l++
+	}
+
+	if l > ColferSizeMax {
+		return l, ColferMax(fmt.Sprintf("colfer: struct model.course exceeds %d bytes", ColferSizeMax))
+	}
+	return l, nil
+}
+
+// MarshalBinary encodes o as Colfer conform encoding.BinaryMarshaler.
+// The error return option is model.ColferMax.
+func (o *Course) MarshalBinary() (data []byte, err error) {
+	l, err := o.MarshalLen()
+	if err != nil {
+		return nil, err
+	}
+	data = make([]byte, l)
+	o.MarshalTo(data)
+	return data, nil
+}
+
+// Unmarshal decodes data as Colfer and returns the number of bytes read.
+// The error return options are io.EOF, model.ColferError and model.ColferMax.
+func (o *Course) Unmarshal(data []byte) (int, error) {
+	if len(data) == 0 {
+		return 0, io.EOF
+	}
+	header := data[0]
+	i := 1
+
+	if header == 0 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Created = int64(x)
+
+		header = data[i]
+		i++
+	} else if header == 0|0x80 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Created = int64(^x + 1)
+
+		header = data[i]
+		i++
+	}
+
+	if header == 1 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Modified = int64(x)
+
+		header = data[i]
+		i++
+	} else if header == 1|0x80 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Modified = int64(^x + 1)
+
+		header = data[i]
+		i++
+	}
+
+	if header == 2 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Expires = int64(x)
+
+		header = data[i]
+		i++
+	} else if header == 2|0x80 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Expires = int64(^x + 1)
+
+		header = data[i]
+		i++
+	}
+
+	if header == 3 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Deleted = int64(x)
+
+		header = data[i]
+		i++
+	} else if header == 3|0x80 {
+		if i+1 >= len(data) {
+			i++
+			goto eof
+		}
+		x := uint64(data[i])
+		i++
+
+		if x >= 0x80 {
+			x &= 0x7f
+			for shift := uint(7); ; shift += 7 {
+				b := uint64(data[i])
+				i++
+				if i >= len(data) {
+					goto eof
+				}
+
+				if b < 0x80 || shift == 56 {
+					x |= b << shift
+					break
+				}
+				x |= (b & 0x7f) << shift
+			}
+		}
+		o.Deleted = int64(^x + 1)
+
+		header = data[i]
+		i++
+	}
+
+	if header == 4 {
+		if i >= len(data) {
+			goto eof
+		}
+		o.Secret = true
+		header = data[i]
+		i++
+	}
+
+	if header == 5 {
+		if i >= len(data) {
+			goto eof
+		}
+		o.Cronjewel = true
+		header = data[i]
+		i++
+	}
+
+	if header != 0x7f {
+		return 0, ColferError(i - 1)
+	}
+	if i < ColferSizeMax {
+		return i, nil
+	}
+eof:
+	if i >= ColferSizeMax {
+		return 0, ColferMax(fmt.Sprintf("colfer: struct model.course size exceeds %d bytes", ColferSizeMax))
+	}
+	return 0, io.EOF
+}
+
+// UnmarshalBinary decodes data as Colfer conform encoding.BinaryUnmarshaler.
+// The error return options are io.EOF, model.ColferError, model.ColferTail and model.ColferMax.
+func (o *Course) UnmarshalBinary(data []byte) error {
+	i, err := o.Unmarshal(data)
+	if i < len(data) && err == nil {
+		return ColferTail(i)
+	}
+	return err
+}

From efabb291d794ccc887c6e1139d36be7e47f35fdd Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 4 Sep 2018 17:09:32 +0200
Subject: [PATCH 06/20] Work on database revamp [WIP]

---
 database/base.go                    |  63 ----
 database/database.go                |  57 ++--
 database/databases.go               |  56 ++++
 database/dbutils/wrapper.go         |  67 ----
 database/interface.go               |  17 +
 database/model.go                   |   2 -
 database/model/base.go              |  47 +++
 database/model/formats.go           |  15 +
 database/model/key.go               |  14 +
 database/model/meta-bench_test.go   | 466 ++++++++++++++++++++++++++++
 database/model/meta-gencode.go      | 157 ++++++++++
 database/model/meta-gencode_test.go |  35 +++
 database/model/meta.gencode.gen.go  | 163 ----------
 database/model/meta.go              |  74 ++++-
 database/model/meta_test.go         | 311 -------------------
 database/model/model.go             |   7 +
 database/model/wrapper.go           |  67 ++++
 database/storage/interface.go       |  42 +--
 database/storages.go                |  36 +++
 19 files changed, 1032 insertions(+), 664 deletions(-)
 delete mode 100644 database/base.go
 create mode 100644 database/databases.go
 delete mode 100644 database/dbutils/wrapper.go
 create mode 100644 database/interface.go
 create mode 100644 database/model/base.go
 create mode 100644 database/model/formats.go
 create mode 100644 database/model/key.go
 create mode 100644 database/model/meta-bench_test.go
 create mode 100644 database/model/meta-gencode.go
 create mode 100644 database/model/meta-gencode_test.go
 delete mode 100644 database/model/meta.gencode.gen.go
 delete mode 100644 database/model/meta_test.go
 create mode 100644 database/model/wrapper.go
 create mode 100644 database/storages.go

diff --git a/database/base.go b/database/base.go
deleted file mode 100644
index a9bf424..0000000
--- a/database/base.go
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"errors"
-	"strings"
-
-	"github.com/Safing/safing-core/database/dbutils"
-
-	"github.com/ipfs/go-datastore"
-	uuid "github.com/satori/go.uuid"
-)
-
-type Base struct {
-	dbKey *datastore.Key
-	meta  *dbutils.Meta
-}
-
-func (m *Base) SetKey(key *datastore.Key) {
-	m.dbKey = key
-}
-
-func (m *Base) GetKey() *datastore.Key {
-	return m.dbKey
-}
-
-func (m *Base) FmtKey() string {
-	return m.dbKey.String()
-}
-
-func (m *Base) Meta() *dbutils.Meta {
-	return m.meta
-}
-
-func (m *Base) CreateObject(namespace *datastore.Key, name string, model Model) error {
-	var newKey datastore.Key
-	if name == "" {
-		newKey = NewInstance(namespace.ChildString(getTypeName(model)), strings.Replace(uuid.NewV4().String(), "-", "", -1))
-	} else {
-		newKey = NewInstance(namespace.ChildString(getTypeName(model)), name)
-	}
-	m.dbKey = &newKey
-	return Create(*m.dbKey, model)
-}
-
-func (m *Base) SaveObject(model Model) error {
-	if m.dbKey == nil {
-		return errors.New("cannot save new object, use Create() instead")
-	}
-	return Update(*m.dbKey, model)
-}
-
-func (m *Base) Delete() error {
-	if m.dbKey == nil {
-		return errors.New("cannot delete object unsaved object")
-	}
-	return Delete(*m.dbKey)
-}
-
-func NewInstance(k datastore.Key, s string) datastore.Key {
-	return datastore.NewKey(k.String() + ":" + s)
-}
diff --git a/database/database.go b/database/database.go
index b1b7a5f..863604e 100644
--- a/database/database.go
+++ b/database/database.go
@@ -9,29 +9,20 @@ import (
 	"path"
 	"strings"
 
-	ds "github.com/ipfs/go-datastore"
-	dsq "github.com/ipfs/go-datastore/query"
-	mount "github.com/ipfs/go-datastore/syncmount"
-
 	"github.com/Safing/safing-core/database/dbutils"
-	"github.com/Safing/safing-core/database/ds/channelshim"
-	"github.com/Safing/safing-core/database/ds/leveldb"
 	"github.com/Safing/safing-core/log"
 	"github.com/Safing/safing-core/meta"
 )
 
-// TODO: do not let other modules panic, even if database module crashes.
-var db ds.Datastore
-
 var ErrNotFound = errors.New("database: entry could not be found")
 
 func init() {
-	if strings.HasSuffix(os.Args[0], ".test") {
-		// testing setup
-		log.Warning("===== DATABASE RUNNING IN TEST MODE =====")
-		db = channelshim.NewChanneledDatastore(ds.NewMapDatastore())
-		return
-	}
+	// if strings.HasSuffix(os.Args[0], ".test") {
+	// 	// testing setup
+	// 	log.Warning("===== DATABASE RUNNING IN TEST MODE =====")
+	// 	db = channelshim.NewChanneledDatastore(ds.NewMapDatastore())
+	// 	return
+	// }
 
 	// sfsDB, err := simplefs.NewDatastore(meta.DatabaseDir())
 	// if err != nil {
@@ -39,24 +30,24 @@ func init() {
 	// 	os.Exit(1)
 	// }
 
-	ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err)
-		os.Exit(1)
-	}
-
-	mapDB := ds.NewMapDatastore()
-
-	db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{
-		mount.Mount{
-			Prefix:    ds.NewKey("/Run"),
-			Datastore: mapDB,
-		},
-		mount.Mount{
-			Prefix:    ds.NewKey("/"),
-			Datastore: ldb,
-		},
-	}))
+	// ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{})
+	// if err != nil {
+	// 	fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err)
+	// 	os.Exit(1)
+	// }
+	//
+	// mapDB := ds.NewMapDatastore()
+	//
+	// db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{
+	// 	mount.Mount{
+	// 		Prefix:    ds.NewKey("/Run"),
+	// 		Datastore: mapDB,
+	// 	},
+	// 	mount.Mount{
+	// 		Prefix:    ds.NewKey("/"),
+	// 		Datastore: ldb,
+	// 	},
+	// }))
 
 }
 
diff --git a/database/databases.go b/database/databases.go
new file mode 100644
index 0000000..d4643f5
--- /dev/null
+++ b/database/databases.go
@@ -0,0 +1,56 @@
+package database
+
+
+var (
+  databases = make(map[string]*storage.Interface)
+  databasesLock sync.Mutex
+)
+
+func getDatabase(name string) *storage.Interface {
+  databasesLock.Lock()
+  defer   databasesLock.Unlock()
+  storage, ok := databases[name]
+  if ok {
+    return
+  }
+}
+
+func databaseExists(name string) (exists bool) {
+  // check if folder exists
+  return true
+}
+
+// CreateDatabase creates a new database with given name and type.
+func CreateDatabase(name string, storageType string) error {
+  databasesLock.Lock()
+  defer   databasesLock.Unlock()
+  _, ok := databases[name]
+  if ok {
+    return errors.New("database with this name already loaded.")
+  }
+  if databaseExists(name) {
+    return errors.New("database with this name already exists.")
+  }
+
+  iface, err := startDatabase(name)
+  if err != nil {
+    return err
+  }
+  databases[name] = iface
+  return nil
+}
+
+// InjectDatabase injects an already running database into the system.
+func InjectDatabase(name string, iface *storage.Interface) error {
+  databasesLock.Lock()
+  defer   databasesLock.Unlock()
+  _, ok := databases[name]
+  if ok {
+    return errors.New("database with this name already loaded.")
+  }
+  if databaseExists(name) {
+    return errors.New("database with this name already exists.")
+  }
+  databases[name] = iface
+  return nil
+}
diff --git a/database/dbutils/wrapper.go b/database/dbutils/wrapper.go
deleted file mode 100644
index 98ef68c..0000000
--- a/database/dbutils/wrapper.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-/*
-Package dbutils provides important function for datastore backends without creating an import loop.
-*/
-package dbutils
-
-import (
-	"errors"
-	"fmt"
-
-	"github.com/ipfs/go-datastore"
-
-	"github.com/Safing/safing-core/formats/dsd"
-	"github.com/Safing/safing-core/formats/varint"
-)
-
-type Wrapper struct {
-	dbKey  *datastore.Key
-	meta   *Meta
-	Format uint8
-	Data   []byte
-}
-
-func NewWrapper(key *datastore.Key, data []byte) (*Wrapper, error) {
-	// line crashes with: panic: runtime error: index out of range
-	format, _, err := varint.Unpack8(data)
-	if err != nil {
-		return nil, fmt.Errorf("database: could not get dsd format: %s", err)
-	}
-
-	new := &Wrapper{
-		Format: format,
-		Data:   data,
-	}
-	new.SetKey(key)
-
-	return new, nil
-}
-
-func (w *Wrapper) SetKey(key *datastore.Key) {
-	w.dbKey = key
-}
-
-func (w *Wrapper) GetKey() *datastore.Key {
-	return w.dbKey
-}
-
-func (w *Wrapper) FmtKey() string {
-	return w.dbKey.String()
-}
-
-func DumpModel(uncertain interface{}, storageType uint8) ([]byte, error) {
-	wrapped, ok := uncertain.(*Wrapper)
-	if ok {
-		if storageType != dsd.AUTO && storageType != wrapped.Format {
-			return nil, errors.New("could not dump model, wrapped object format mismatch")
-		}
-		return wrapped.Data, nil
-	}
-
-	dumped, err := dsd.Dump(uncertain, storageType)
-	if err != nil {
-		return nil, err
-	}
-	return dumped, nil
-}
diff --git a/database/interface.go b/database/interface.go
new file mode 100644
index 0000000..2d249ba
--- /dev/null
+++ b/database/interface.go
@@ -0,0 +1,17 @@
+package database
+
+type Interface struct {
+  local bool
+  internal bool
+}
+
+func NewInterface(local bool, internal bool) *Interface {
+  return &Interface{
+    local: local,
+    internal: internal,
+  }
+}
+
+func (i *Interface) Get(string key) (model.Model, error) {
+  return nil, nil
+}
diff --git a/database/model.go b/database/model.go
index 4283ba2..2644043 100644
--- a/database/model.go
+++ b/database/model.go
@@ -7,8 +7,6 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/ipfs/go-datastore"
-
 	"github.com/Safing/safing-core/database/dbutils"
 	"github.com/Safing/safing-core/formats/dsd"
 )
diff --git a/database/model/base.go b/database/model/base.go
new file mode 100644
index 0000000..5ad252d
--- /dev/null
+++ b/database/model/base.go
@@ -0,0 +1,47 @@
+package model
+
+import (
+	"github.com/Safing/portbase/formats/dsd"
+)
+
+// Base provides a quick way to comply with the Model interface.
+type Base struct {
+	dbName string
+	dbKey  string
+	meta   *Meta
+}
+
+// Key returns the key of the database record.
+func (b *Base) Key() string {
+	return b.dbKey
+}
+
+// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
+func (b *Base) SetKey(key string) {
+	b.dbKey = key
+}
+
+// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
+func (b *Base) MoveTo(key string) {
+	b.dbKey = key
+	b.meta.Reset()
+}
+
+// Meta returns the metadata object for this record.
+func (b *Base) Meta() *Meta {
+	return b.meta
+}
+
+// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
+func (b *Base) SetMeta(meta *Meta) {
+	b.meta = meta
+}
+
+// Marshal marshals the object, without the database key or metadata
+func (b *Base) Marshal(format uint8) ([]byte, error) {
+	dumped, err := dsd.Dump(b, format)
+	if err != nil {
+		return nil, err
+	}
+	return dumped, nil
+}
diff --git a/database/model/formats.go b/database/model/formats.go
new file mode 100644
index 0000000..6718593
--- /dev/null
+++ b/database/model/formats.go
@@ -0,0 +1,15 @@
+package model
+
+import (
+	"github.com/Safing/portbase/formats/dsd"
+)
+
+// Reimport DSD storage types
+const (
+	AUTO    = dsd.AUTO
+	STRING  = dsd.STRING  // S
+	BYTES   = dsd.BYTES   // X
+	JSON    = dsd.JSON    // J
+	BSON    = dsd.BSON    // B
+	GenCode = dsd.GenCode // G (reserved)
+)
diff --git a/database/model/key.go b/database/model/key.go
new file mode 100644
index 0000000..19c7c59
--- /dev/null
+++ b/database/model/key.go
@@ -0,0 +1,14 @@
+package model
+
+import (
+	"errors"
+	"strings"
+)
+
+func ParseKey(key string) (dbName, dbKey string, err error) {
+	splitted := strings.SplitN(key, ":", 2)
+	if len(splitted) == 2 {
+		return splitted[0], splitted[1], nil
+	}
+	return "", "", errors.New("invalid key")
+}
diff --git a/database/model/meta-bench_test.go b/database/model/meta-bench_test.go
new file mode 100644
index 0000000..262aaec
--- /dev/null
+++ b/database/model/meta-bench_test.go
@@ -0,0 +1,466 @@
+package model
+
+// Benchmark:
+// BenchmarkAllocateBytes-8                	2000000000	         0.76 ns/op
+// BenchmarkAllocateStruct1-8              	2000000000	         0.76 ns/op
+// BenchmarkAllocateStruct2-8              	2000000000	         0.79 ns/op
+// BenchmarkMetaSerializeContainer-8       	 1000000	      1703 ns/op
+// BenchmarkMetaUnserializeContainer-8     	 2000000	       950 ns/op
+// BenchmarkMetaSerializeVarInt-8          	 3000000	       457 ns/op
+// BenchmarkMetaUnserializeVarInt-8        	20000000	        62.9 ns/op
+// BenchmarkMetaSerializeWithXDR2-8        	 1000000	      2360 ns/op
+// BenchmarkMetaUnserializeWithXDR2-8      	  500000	      3189 ns/op
+// BenchmarkMetaSerializeWithColfer-8      	10000000	       237 ns/op
+// BenchmarkMetaUnserializeWithColfer-8    	20000000	        51.7 ns/op
+// BenchmarkMetaSerializeWithCodegen-8     	50000000	        23.7 ns/op
+// BenchmarkMetaUnserializeWithCodegen-8   	100000000	        18.9 ns/op
+// BenchmarkMetaSerializeWithDSDJSON-8     	 1000000	      2398 ns/op
+// BenchmarkMetaUnserializeWithDSDJSON-8   	  300000	      6264 ns/op
+
+import (
+	"testing"
+	"time"
+
+	"github.com/Safing/portbase/container"
+	"github.com/Safing/portbase/formats/dsd"
+	"github.com/Safing/portbase/formats/varint"
+	// Colfer
+	// "github.com/Safing/portbase/database/model/model"
+	// XDR
+	// xdr2 "github.com/davecgh/go-xdr/xdr2"
+)
+
+var (
+	testMeta = &Meta{
+		created:   time.Now().Unix(),
+		modified:  time.Now().Unix(),
+		expires:   time.Now().Unix(),
+		deleted:   time.Now().Unix(),
+		secret:    true,
+		cronjewel: true,
+	}
+)
+
+func BenchmarkAllocateBytes(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = make([]byte, 33)
+	}
+}
+
+func BenchmarkAllocateStruct1(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var new Meta
+		_ = new
+	}
+}
+
+func BenchmarkAllocateStruct2(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = Meta{}
+	}
+}
+
+func BenchmarkMetaSerializeContainer(b *testing.B) {
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		c := container.New()
+		c.AppendNumber(uint64(testMeta.created))
+		c.AppendNumber(uint64(testMeta.modified))
+		c.AppendNumber(uint64(testMeta.expires))
+		c.AppendNumber(uint64(testMeta.deleted))
+		switch {
+		case testMeta.secret && testMeta.cronjewel:
+			c.AppendNumber(3)
+		case testMeta.secret:
+			c.AppendNumber(1)
+		case testMeta.cronjewel:
+			c.AppendNumber(2)
+		default:
+			c.AppendNumber(0)
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeContainer(b *testing.B) {
+
+	// Setup
+	c := container.New()
+	c.AppendNumber(uint64(testMeta.created))
+	c.AppendNumber(uint64(testMeta.modified))
+	c.AppendNumber(uint64(testMeta.expires))
+	c.AppendNumber(uint64(testMeta.deleted))
+	switch {
+	case testMeta.secret && testMeta.cronjewel:
+		c.AppendNumber(3)
+	case testMeta.secret:
+		c.AppendNumber(1)
+	case testMeta.cronjewel:
+		c.AppendNumber(2)
+	default:
+		c.AppendNumber(0)
+	}
+	encodedData := c.CompileData()
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		var err error
+		var num uint64
+		c := container.New(encodedData)
+		num, err = c.GetNextN64()
+		newMeta.created = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.modified = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.expires = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.deleted = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+
+		flags, err := c.GetNextN8()
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+
+		switch flags {
+		case 3:
+			newMeta.secret = true
+			newMeta.cronjewel = true
+		case 2:
+			newMeta.cronjewel = true
+		case 1:
+			newMeta.secret = true
+		case 0:
+		default:
+			b.Errorf("invalid flag value: %d", flags)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaSerializeVarInt(b *testing.B) {
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		encoded := make([]byte, 33)
+		offset := 0
+		data := varint.Pack64(uint64(testMeta.created))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+		data = varint.Pack64(uint64(testMeta.modified))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+		data = varint.Pack64(uint64(testMeta.expires))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+		data = varint.Pack64(uint64(testMeta.deleted))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+
+		switch {
+		case testMeta.secret && testMeta.cronjewel:
+			encoded[offset] = 3
+		case testMeta.secret:
+			encoded[offset] = 1
+		case testMeta.cronjewel:
+			encoded[offset] = 2
+		default:
+			encoded[offset] = 0
+		}
+		offset++
+	}
+
+}
+
+func BenchmarkMetaUnserializeVarInt(b *testing.B) {
+
+	// Setup
+	encoded := make([]byte, 33)
+	offset := 0
+	data := varint.Pack64(uint64(testMeta.created))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+	data = varint.Pack64(uint64(testMeta.modified))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+	data = varint.Pack64(uint64(testMeta.expires))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+	data = varint.Pack64(uint64(testMeta.deleted))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+
+	switch {
+	case testMeta.secret && testMeta.cronjewel:
+		encoded[offset] = 3
+	case testMeta.secret:
+		encoded[offset] = 1
+	case testMeta.cronjewel:
+		encoded[offset] = 2
+	default:
+		encoded[offset] = 0
+	}
+	offset++
+	encodedData := encoded[:offset]
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		offset = 0
+
+		num, n, err := varint.Unpack64(encodedData)
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.created = int64(num)
+		offset += n
+
+		num, n, err = varint.Unpack64(encodedData[offset:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.modified = int64(num)
+		offset += n
+
+		num, n, err = varint.Unpack64(encodedData[offset:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.expires = int64(num)
+		offset += n
+
+		num, n, err = varint.Unpack64(encodedData[offset:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.deleted = int64(num)
+		offset += n
+
+		switch encodedData[offset] {
+		case 3:
+			newMeta.secret = true
+			newMeta.cronjewel = true
+		case 2:
+			newMeta.cronjewel = true
+		case 1:
+			newMeta.secret = true
+		case 0:
+		default:
+			b.Errorf("invalid flag value: %d", encodedData[offset])
+			return
+		}
+	}
+
+}
+
+// func BenchmarkMetaSerializeWithXDR2(b *testing.B) {
+//
+// 	// Setup
+// 	var w bytes.Buffer
+//
+// 	// Reset timer for precise results
+// 	b.ResetTimer()
+//
+// 	// Start benchmark
+// 	for i := 0; i < b.N; i++ {
+// 		w.Reset()
+// 		_, err := xdr2.Marshal(&w, testMeta)
+// 		if err != nil {
+// 			b.Errorf("failed to serialize with xdr2: %s", err)
+// 			return
+// 		}
+// 	}
+//
+// }
+
+// func BenchmarkMetaUnserializeWithXDR2(b *testing.B) {
+//
+// 	// Setup
+// 	var w bytes.Buffer
+// 	_, err := xdr2.Marshal(&w, testMeta)
+// 	if err != nil {
+// 		b.Errorf("failed to serialize with xdr2: %s", err)
+// 	}
+// 	encodedData := w.Bytes()
+//
+// 	// Reset timer for precise results
+// 	b.ResetTimer()
+//
+// 	// Start benchmark
+// 	for i := 0; i < b.N; i++ {
+// 		var newMeta Meta
+// 		_, err := xdr2.Unmarshal(bytes.NewReader(encodedData), &newMeta)
+// 		if err != nil {
+// 			b.Errorf("failed to unserialize with xdr2: %s", err)
+// 			return
+// 		}
+// 	}
+//
+// }
+
+// func BenchmarkMetaSerializeWithColfer(b *testing.B) {
+//
+// 	testColf := &model.Course{
+// 		Created:   time.Now().Unix(),
+// 		Modified:  time.Now().Unix(),
+// 		Expires:   time.Now().Unix(),
+// 		Deleted:   time.Now().Unix(),
+// 		Secret:    true,
+// 		Cronjewel: true,
+// 	}
+//
+// 	// Setup
+// 	for i := 0; i < b.N; i++ {
+// 		_, err := testColf.MarshalBinary()
+// 		if err != nil {
+// 			b.Errorf("failed to serialize with colfer: %s", err)
+// 			return
+// 		}
+// 	}
+//
+// }
+
+// func BenchmarkMetaUnserializeWithColfer(b *testing.B) {
+//
+// 	testColf := &model.Course{
+// 		Created:   time.Now().Unix(),
+// 		Modified:  time.Now().Unix(),
+// 		Expires:   time.Now().Unix(),
+// 		Deleted:   time.Now().Unix(),
+// 		Secret:    true,
+// 		Cronjewel: true,
+// 	}
+// 	encodedData, err := testColf.MarshalBinary()
+// 	if err != nil {
+// 		b.Errorf("failed to serialize with colfer: %s", err)
+// 		return
+// 	}
+//
+// 	// Setup
+// 	for i := 0; i < b.N; i++ {
+// 		var testUnColf model.Course
+// 		err := testUnColf.UnmarshalBinary(encodedData)
+// 		if err != nil {
+// 			b.Errorf("failed to unserialize with colfer: %s", err)
+// 			return
+// 		}
+// 	}
+//
+// }
+
+func BenchmarkMetaSerializeWithCodegen(b *testing.B) {
+
+	for i := 0; i < b.N; i++ {
+		_, err := testMeta.GenCodeMarshal(nil)
+		if err != nil {
+			b.Errorf("failed to serialize with codegen: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
+
+	// Setup
+	encodedData, err := testMeta.GenCodeMarshal(nil)
+	if err != nil {
+		b.Errorf("failed to serialize with codegen: %s", err)
+		return
+	}
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		_, err := newMeta.GenCodeUnmarshal(encodedData)
+		if err != nil {
+			b.Errorf("failed to unserialize with codegen: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
+
+	for i := 0; i < b.N; i++ {
+		_, err := dsd.Dump(testMeta, dsd.JSON)
+		if err != nil {
+			b.Errorf("failed to serialize with DSD/JSON: %s", err)
+			return
+		}
+	}
+
+}
+
+func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
+
+	// Setup
+	encodedData, err := dsd.Dump(testMeta, dsd.JSON)
+	if err != nil {
+		b.Errorf("failed to serialize with DSD/JSON: %s", err)
+		return
+	}
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for i := 0; i < b.N; i++ {
+		var newMeta Meta
+		_, err := dsd.Load(encodedData, &newMeta)
+		if err != nil {
+			b.Errorf("failed to unserialize with DSD/JSON: %s", err)
+			return
+		}
+	}
+
+}
diff --git a/database/model/meta-gencode.go b/database/model/meta-gencode.go
new file mode 100644
index 0000000..6c4e76f
--- /dev/null
+++ b/database/model/meta-gencode.go
@@ -0,0 +1,157 @@
+package model
+
+import (
+	"io"
+	"time"
+	"unsafe"
+)
+
+var (
+	_ = unsafe.Sizeof(0)
+	_ = io.ReadFull
+	_ = time.Now()
+)
+
+// GenCodeSize returns the size of the gencode marshalled byte slice
+func (d *Meta) GenCodeSize() (s uint64) {
+	s += 34
+	return
+}
+
+// GenCodeMarshal gencode marshalls Meta into the given byte array, or a new one if its too small.
+func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
+	size := d.GenCodeSize()
+	{
+		if uint64(cap(buf)) >= size {
+			buf = buf[:size]
+		} else {
+			buf = make([]byte, size)
+		}
+	}
+	i := uint64(0)
+
+	{
+
+		buf[0+0] = byte(d.created >> 0)
+
+		buf[1+0] = byte(d.created >> 8)
+
+		buf[2+0] = byte(d.created >> 16)
+
+		buf[3+0] = byte(d.created >> 24)
+
+		buf[4+0] = byte(d.created >> 32)
+
+		buf[5+0] = byte(d.created >> 40)
+
+		buf[6+0] = byte(d.created >> 48)
+
+		buf[7+0] = byte(d.created >> 56)
+
+	}
+	{
+
+		buf[0+8] = byte(d.modified >> 0)
+
+		buf[1+8] = byte(d.modified >> 8)
+
+		buf[2+8] = byte(d.modified >> 16)
+
+		buf[3+8] = byte(d.modified >> 24)
+
+		buf[4+8] = byte(d.modified >> 32)
+
+		buf[5+8] = byte(d.modified >> 40)
+
+		buf[6+8] = byte(d.modified >> 48)
+
+		buf[7+8] = byte(d.modified >> 56)
+
+	}
+	{
+
+		buf[0+16] = byte(d.expires >> 0)
+
+		buf[1+16] = byte(d.expires >> 8)
+
+		buf[2+16] = byte(d.expires >> 16)
+
+		buf[3+16] = byte(d.expires >> 24)
+
+		buf[4+16] = byte(d.expires >> 32)
+
+		buf[5+16] = byte(d.expires >> 40)
+
+		buf[6+16] = byte(d.expires >> 48)
+
+		buf[7+16] = byte(d.expires >> 56)
+
+	}
+	{
+
+		buf[0+24] = byte(d.deleted >> 0)
+
+		buf[1+24] = byte(d.deleted >> 8)
+
+		buf[2+24] = byte(d.deleted >> 16)
+
+		buf[3+24] = byte(d.deleted >> 24)
+
+		buf[4+24] = byte(d.deleted >> 32)
+
+		buf[5+24] = byte(d.deleted >> 40)
+
+		buf[6+24] = byte(d.deleted >> 48)
+
+		buf[7+24] = byte(d.deleted >> 56)
+
+	}
+	{
+		if d.secret {
+			buf[32] = 1
+		} else {
+			buf[32] = 0
+		}
+	}
+	{
+		if d.cronjewel {
+			buf[33] = 1
+		} else {
+			buf[33] = 0
+		}
+	}
+	return buf[:i+34], nil
+}
+
+// GenCodeUnmarshal gencode unmarshalls Meta and returns the bytes read.
+func (d *Meta) GenCodeUnmarshal(buf []byte) (uint64, error) {
+	i := uint64(0)
+
+	{
+
+		d.created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
+
+	}
+	{
+
+		d.modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
+
+	}
+	{
+
+		d.expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
+
+	}
+	{
+
+		d.deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
+
+	}
+	{
+		d.secret = buf[32] == 1
+	}
+	{
+		d.cronjewel = buf[33] == 1
+	}
+	return i + 34, nil
+}
diff --git a/database/model/meta-gencode_test.go b/database/model/meta-gencode_test.go
new file mode 100644
index 0000000..4f3bdcb
--- /dev/null
+++ b/database/model/meta-gencode_test.go
@@ -0,0 +1,35 @@
+package model
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+var (
+	genCodeTestMeta = &Meta{
+		created:   time.Now().Unix(),
+		modified:  time.Now().Unix(),
+		expires:   time.Now().Unix(),
+		deleted:   time.Now().Unix(),
+		secret:    true,
+		cronjewel: true,
+	}
+)
+
+func TestGenCode(t *testing.T) {
+	encoded, err := genCodeTestMeta.GenCodeMarshal(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	new := &Meta{}
+	_, err = new.GenCodeUnmarshal(encoded)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(genCodeTestMeta, new) {
+		t.Errorf("objects are not equal, got: %v", new)
+	}
+}
diff --git a/database/model/meta.gencode.gen.go b/database/model/meta.gencode.gen.go
deleted file mode 100644
index 4cf07f0..0000000
--- a/database/model/meta.gencode.gen.go
+++ /dev/null
@@ -1,163 +0,0 @@
-package model
-
-import (
-	"io"
-	"time"
-	"unsafe"
-)
-
-var (
-	_ = unsafe.Sizeof(0)
-	_ = io.ReadFull
-	_ = time.Now()
-)
-
-// type Meta struct {
-// 	Created   int64
-// 	Modified  int64
-// 	Expires   int64
-// 	Deleted   int64
-// 	Secret    bool
-// 	Cronjewel bool
-// }
-
-func (d *Meta) Size() (s uint64) {
-
-	s += 34
-	return
-}
-func (d *Meta) Marshal(buf []byte) ([]byte, error) {
-	size := d.Size()
-	{
-		if uint64(cap(buf)) >= size {
-			buf = buf[:size]
-		} else {
-			buf = make([]byte, size)
-		}
-	}
-	i := uint64(0)
-
-	{
-
-		buf[0+0] = byte(d.Created >> 0)
-
-		buf[1+0] = byte(d.Created >> 8)
-
-		buf[2+0] = byte(d.Created >> 16)
-
-		buf[3+0] = byte(d.Created >> 24)
-
-		buf[4+0] = byte(d.Created >> 32)
-
-		buf[5+0] = byte(d.Created >> 40)
-
-		buf[6+0] = byte(d.Created >> 48)
-
-		buf[7+0] = byte(d.Created >> 56)
-
-	}
-	{
-
-		buf[0+8] = byte(d.Modified >> 0)
-
-		buf[1+8] = byte(d.Modified >> 8)
-
-		buf[2+8] = byte(d.Modified >> 16)
-
-		buf[3+8] = byte(d.Modified >> 24)
-
-		buf[4+8] = byte(d.Modified >> 32)
-
-		buf[5+8] = byte(d.Modified >> 40)
-
-		buf[6+8] = byte(d.Modified >> 48)
-
-		buf[7+8] = byte(d.Modified >> 56)
-
-	}
-	{
-
-		buf[0+16] = byte(d.Expires >> 0)
-
-		buf[1+16] = byte(d.Expires >> 8)
-
-		buf[2+16] = byte(d.Expires >> 16)
-
-		buf[3+16] = byte(d.Expires >> 24)
-
-		buf[4+16] = byte(d.Expires >> 32)
-
-		buf[5+16] = byte(d.Expires >> 40)
-
-		buf[6+16] = byte(d.Expires >> 48)
-
-		buf[7+16] = byte(d.Expires >> 56)
-
-	}
-	{
-
-		buf[0+24] = byte(d.Deleted >> 0)
-
-		buf[1+24] = byte(d.Deleted >> 8)
-
-		buf[2+24] = byte(d.Deleted >> 16)
-
-		buf[3+24] = byte(d.Deleted >> 24)
-
-		buf[4+24] = byte(d.Deleted >> 32)
-
-		buf[5+24] = byte(d.Deleted >> 40)
-
-		buf[6+24] = byte(d.Deleted >> 48)
-
-		buf[7+24] = byte(d.Deleted >> 56)
-
-	}
-	{
-		if d.Secret {
-			buf[32] = 1
-		} else {
-			buf[32] = 0
-		}
-	}
-	{
-		if d.Cronjewel {
-			buf[33] = 1
-		} else {
-			buf[33] = 0
-		}
-	}
-	return buf[:i+34], nil
-}
-
-func (d *Meta) Unmarshal(buf []byte) (uint64, error) {
-	i := uint64(0)
-
-	{
-
-		d.Created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
-
-	}
-	{
-
-		d.Modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
-
-	}
-	{
-
-		d.Expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
-
-	}
-	{
-
-		d.Deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
-
-	}
-	{
-		d.Secret = buf[32] == 1
-	}
-	{
-		d.Cronjewel = buf[33] == 1
-	}
-	return i + 34, nil
-}
diff --git a/database/model/meta.go b/database/model/meta.go
index 161ee64..2424821 100644
--- a/database/model/meta.go
+++ b/database/model/meta.go
@@ -1,10 +1,72 @@
 package model
 
+import "time"
+
+// Meta holds
 type Meta struct {
-	Created   int64 `json:"c,omitempty" bson:"c,omitempty"`
-	Modified  int64 `json:"m,omitempty" bson:"m,omitempty"`
-	Expires   int64 `json:"e,omitempty" bson:"e,omitempty"`
-	Deleted   int64 `json:"d,omitempty" bson:"d,omitempty"`
-	Secret    bool  `json:"s,omitempty" bson:"s,omitempty"` // secrets must not be sent to the UI, only synced between nodes
-	Cronjewel bool  `json:"j,omitempty" bson:"j,omitempty"` // crownjewels must never leave the instance, but may be read by the UI
+	created   int64
+	modified  int64
+	expires   int64
+	deleted   int64
+	secret    bool // secrets must not be sent to the UI, only synced between nodes
+	cronjewel bool // crownjewels must never leave the instance, but may be read by the UI
+}
+
+// SetAbsoluteExpiry sets an absolute expiry time, that is not affected when the record is updated.
+func (m *Meta) SetAbsoluteExpiry(time int64) {
+	m.expires = time
+	m.deleted = 0
+}
+
+// SetRelativateExpiry sets a relative expiry that is automatically updated whenever the record is updated/saved.
+func (m *Meta) SetRelativateExpiry(duration int64) {
+	if duration >= 0 {
+		m.deleted = -duration
+	}
+}
+
+// MakeCrownJewel marks the database records as a crownjewel, meaning that it will not be sent/synced to other devices.
+func (m *Meta) MakeCrownJewel() {
+	m.cronjewel = true
+}
+
+// MakeSecret sets the database record as secret, meaning that it may only be used internally, and not by interfacing processes, such as the UI.
+func (m *Meta) MakeSecret() {
+	m.secret = true
+}
+
+// Update updates the internal meta states and should be called before writing the record to the database.
+func (m *Meta) Update() {
+	now := time.Now().Unix()
+	m.modified = now
+	if m.created == 0 {
+		m.created = now
+	}
+	if m.deleted < 0 {
+		m.expires = now - m.deleted
+	}
+}
+
+// Reset resets all metadata, except for the secret and crownjewel status.
+func (m *Meta) Reset() {
+	m.created = 0
+	m.modified = 0
+	m.expires = 0
+	m.deleted = 0
+}
+
+// CheckScope checks whether the current database record exists for the given scope.
+func (m *Meta) CheckScope(now int64, local, internal bool) (recordExists bool) {
+	switch {
+	case m.deleted > 0:
+		return false
+	case m.expires < now:
+		return false
+	case !local && m.cronjewel:
+		return false
+	case !internal && m.secret:
+		return false
+	default:
+		return true
+	}
 }
diff --git a/database/model/meta_test.go b/database/model/meta_test.go
deleted file mode 100644
index 4141fce..0000000
--- a/database/model/meta_test.go
+++ /dev/null
@@ -1,311 +0,0 @@
-package model
-
-import (
-	"bytes"
-	"testing"
-	"time"
-
-	"github.com/Safing/portbase/container"
-	"github.com/Safing/portbase/database/model/model"
-	"github.com/Safing/portbase/formats/dsd"
-	xdr2 "github.com/davecgh/go-xdr/xdr2"
-)
-
-var (
-	testMeta = &Meta{
-		Created:   time.Now().Unix(),
-		Modified:  time.Now().Unix(),
-		Expires:   time.Now().Unix(),
-		Deleted:   time.Now().Unix(),
-		Secret:    true,
-		Cronjewel: true,
-	}
-)
-
-func BenchmarkAllocateBytes(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		_ = make([]byte, 33)
-	}
-}
-
-func BenchmarkAllocateStruct1(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		var new Meta
-		_ = new
-	}
-}
-
-func BenchmarkAllocateStruct2(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		_ = Meta{}
-	}
-}
-
-func BenchmarkMetaSerializeCustom(b *testing.B) {
-
-	// Start benchmark
-	for i := 0; i < b.N; i++ {
-		c := container.New()
-		c.AppendNumber(uint64(testMeta.Created))
-		c.AppendNumber(uint64(testMeta.Modified))
-		c.AppendNumber(uint64(testMeta.Expires))
-		c.AppendNumber(uint64(testMeta.Deleted))
-		switch {
-		case testMeta.Secret && testMeta.Cronjewel:
-			c.AppendNumber(3)
-		case testMeta.Secret:
-			c.AppendNumber(1)
-		case testMeta.Cronjewel:
-			c.AppendNumber(2)
-		default:
-			c.AppendNumber(0)
-		}
-	}
-
-}
-
-func BenchmarkMetaUnserializeCustom(b *testing.B) {
-
-	// Setup
-	c := container.New()
-	c.AppendNumber(uint64(testMeta.Created))
-	c.AppendNumber(uint64(testMeta.Modified))
-	c.AppendNumber(uint64(testMeta.Expires))
-	c.AppendNumber(uint64(testMeta.Deleted))
-	switch {
-	case testMeta.Secret && testMeta.Cronjewel:
-		c.AppendNumber(3)
-	case testMeta.Secret:
-		c.AppendNumber(1)
-	case testMeta.Cronjewel:
-		c.AppendNumber(2)
-	default:
-		c.AppendNumber(0)
-	}
-	encodedData := c.CompileData()
-
-	// Reset timer for precise results
-	b.ResetTimer()
-
-	// Start benchmark
-	for i := 0; i < b.N; i++ {
-		var newMeta Meta
-		var err error
-		var num uint64
-		c := container.New(encodedData)
-		num, err = c.GetNextN64()
-		newMeta.Created = int64(num)
-		if err != nil {
-			b.Errorf("could not decode: %s", err)
-			return
-		}
-		num, err = c.GetNextN64()
-		newMeta.Modified = int64(num)
-		if err != nil {
-			b.Errorf("could not decode: %s", err)
-			return
-		}
-		num, err = c.GetNextN64()
-		newMeta.Expires = int64(num)
-		if err != nil {
-			b.Errorf("could not decode: %s", err)
-			return
-		}
-		num, err = c.GetNextN64()
-		newMeta.Deleted = int64(num)
-		if err != nil {
-			b.Errorf("could not decode: %s", err)
-			return
-		}
-
-		flags, err := c.GetNextN8()
-		if err != nil {
-			b.Errorf("could not decode: %s", err)
-			return
-		}
-
-		switch flags {
-		case 3:
-			newMeta.Secret = true
-			newMeta.Cronjewel = true
-		case 2:
-			newMeta.Cronjewel = true
-		case 1:
-			newMeta.Secret = true
-		case 0:
-		default:
-			b.Errorf("invalid flag value: %d", flags)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaSerializeWithXDR2(b *testing.B) {
-
-	// Setup
-	var w bytes.Buffer
-
-	// Reset timer for precise results
-	b.ResetTimer()
-
-	// Start benchmark
-	for i := 0; i < b.N; i++ {
-		w.Reset()
-		_, err := xdr2.Marshal(&w, testMeta)
-		if err != nil {
-			b.Errorf("failed to serialize with xdr2: %s", err)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaUnserializeWithXDR2(b *testing.B) {
-
-	// Setup
-	var w bytes.Buffer
-	_, err := xdr2.Marshal(&w, testMeta)
-	if err != nil {
-		b.Errorf("failed to serialize with xdr2: %s", err)
-	}
-	encodedData := w.Bytes()
-
-	// Reset timer for precise results
-	b.ResetTimer()
-
-	// Start benchmark
-	for i := 0; i < b.N; i++ {
-		var newMeta Meta
-		_, err := xdr2.Unmarshal(bytes.NewReader(encodedData), &newMeta)
-		if err != nil {
-			b.Errorf("failed to unserialize with xdr2: %s", err)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaSerializeWithColfer(b *testing.B) {
-
-	testColf := &model.Course{
-		Created:   time.Now().Unix(),
-		Modified:  time.Now().Unix(),
-		Expires:   time.Now().Unix(),
-		Deleted:   time.Now().Unix(),
-		Secret:    true,
-		Cronjewel: true,
-	}
-
-	// Setup
-	for i := 0; i < b.N; i++ {
-		_, err := testColf.MarshalBinary()
-		if err != nil {
-			b.Errorf("failed to serialize with colfer: %s", err)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaUnserializeWithColfer(b *testing.B) {
-
-	testColf := &model.Course{
-		Created:   time.Now().Unix(),
-		Modified:  time.Now().Unix(),
-		Expires:   time.Now().Unix(),
-		Deleted:   time.Now().Unix(),
-		Secret:    true,
-		Cronjewel: true,
-	}
-	encodedData, err := testColf.MarshalBinary()
-	if err != nil {
-		b.Errorf("failed to serialize with colfer: %s", err)
-		return
-	}
-
-	// Setup
-	for i := 0; i < b.N; i++ {
-		var testUnColf model.Course
-		err := testUnColf.UnmarshalBinary(encodedData)
-		if err != nil {
-			b.Errorf("failed to unserialize with colfer: %s", err)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaSerializeWithCodegen(b *testing.B) {
-
-	for i := 0; i < b.N; i++ {
-		buf := make([]byte, 34)
-		_, err := testMeta.Marshal(buf)
-		if err != nil {
-			b.Errorf("failed to serialize with codegen: %s", err)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
-
-	// Setup
-	buf := make([]byte, 34)
-	encodedData, err := testMeta.Marshal(buf)
-	if err != nil {
-		b.Errorf("failed to serialize with codegen: %s", err)
-		return
-	}
-
-	// Reset timer for precise results
-	b.ResetTimer()
-
-	// Start benchmark
-	for i := 0; i < b.N; i++ {
-		var newMeta Meta
-		_, err := newMeta.Unmarshal(encodedData)
-		if err != nil {
-			b.Errorf("failed to unserialize with codegen: %s", err)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
-
-	for i := 0; i < b.N; i++ {
-		_, err := dsd.Dump(testMeta, dsd.JSON)
-		if err != nil {
-			b.Errorf("failed to serialize with DSD/JSON: %s", err)
-			return
-		}
-	}
-
-}
-
-func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
-
-	// Setup
-	encodedData, err := dsd.Dump(testMeta, dsd.JSON)
-	if err != nil {
-		b.Errorf("failed to serialize with DSD/JSON: %s", err)
-		return
-	}
-
-	// Reset timer for precise results
-	b.ResetTimer()
-
-	// Start benchmark
-	for i := 0; i < b.N; i++ {
-		var newMeta Meta
-		_, err := dsd.Load(encodedData, &newMeta)
-		if err != nil {
-			b.Errorf("failed to unserialize with DSD/JSON: %s", err)
-			return
-		}
-	}
-
-}
diff --git a/database/model/model.go b/database/model/model.go
index 93debc9..951f6f3 100644
--- a/database/model/model.go
+++ b/database/model/model.go
@@ -1,4 +1,11 @@
 package model
 
+// Model provides an interface for uniformally handling database records.
 type Model interface {
+	Key() string
+	SetKey(key string)
+	MoveTo(key string)
+	Meta() *Meta
+	SetMeta(meta *Meta)
+	Marshal(format uint8) ([]byte, error)
 }
diff --git a/database/model/wrapper.go b/database/model/wrapper.go
new file mode 100644
index 0000000..a19f9d1
--- /dev/null
+++ b/database/model/wrapper.go
@@ -0,0 +1,67 @@
+package model
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/Safing/safing-core/formats/dsd"
+	"github.com/Safing/safing-core/formats/varint"
+)
+
+type Wrapper struct {
+	dbName string
+	dbKey  string
+	meta   *Meta
+	Format uint8
+	Data   []byte
+}
+
+func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
+	format, _, err := varint.Unpack8(data)
+	if err != nil {
+		return nil, fmt.Errorf("database: could not get dsd format: %s", err)
+	}
+
+	new := &Wrapper{
+		dbKey:  key,
+		meta:   meta,
+		Format: format,
+		Data:   data,
+	}
+
+	return new, nil
+}
+
+// Key returns the key of the database record.
+func (w *Wrapper) Key() string {
+	return w.dbKey
+}
+
+// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
+func (w *Wrapper) SetKey(key string) {
+	w.dbKey = key
+}
+
+// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
+func (w *Wrapper) MoveTo(key string) {
+	w.dbKey = key
+	w.meta.Reset()
+}
+
+// Meta returns the metadata object for this record.
+func (w *Wrapper) Meta() *Meta {
+	return w.meta
+}
+
+// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
+func (w *Wrapper) SetMeta(meta *Meta) {
+	w.meta = meta
+}
+
+// Marshal marshals the object, without the database key or metadata
+func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
+	if storageType != dsd.AUTO && storageType != w.Format {
+		return nil, errors.New("could not dump model, wrapped object format mismatch")
+	}
+	return w.Data, nil
+}
diff --git a/database/storage/interface.go b/database/storage/interface.go
index 108a25f..e1fa952 100644
--- a/database/storage/interface.go
+++ b/database/storage/interface.go
@@ -1,30 +1,34 @@
 package storage
 
 import (
-  "github.com/Safing/portbase/database/iterator"
-  "github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/iterator"
+	"github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/query"
 )
 
 // Interface defines the database storage API.
 type Interface interface {
-  // Full
-  Exists(key string) (bool, error)
-  Get(key string) (model.Model, error)
-  Create(key string, model model.Model) error
-  Update(key string, model model.Model) error // create when not exists
-  UpdateOrCreate(key string, model model.Model) error // update, create if not exists.
-  Delete(key string) error
+	// Retrieve
+	Exists(key string) (bool, error)
+	Get(key string) (model.Model, error)
 
-  // Partial
-  // What happens if I mutate a value that does not yet exist? How would I know its type?
-  InsertPartial(key string, partialObject interface{})
-  InsertValue(key string, attribute string, value interface{})
+	// Modify
+	Create(model model.Model) error
+	Update(model model.Model) error         // create when not exists
+	UpdateOrCreate(model model.Model) error // update, create if not exists.
+	Delete(key string) error
 
-  // Query
-  Query(*query.Query) (*iterator.Iterator, error)
+	// Partial
+	// What happens if I mutate a value that does not yet exist? How would I know its type?
+	InsertPartial(key string, partialObject interface{})
+	InsertValue(key string, attribute string, value interface{})
 
-  // Meta
-  LetExpire(key string, timestamp int64) error
-  MakeSecret(key string) error // only visible internal
-  MakeCrownJewel(key string) error // do not sync between devices
+	// Query
+	Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
+
+	// Meta
+	SetAbsoluteExpiry(key string, time int64)
+	SetRelativateExpiry(key string, duration int64)
+	MakeCrownJewel(key string)
+	MakeSecret(key string)
 }
diff --git a/database/storages.go b/database/storages.go
new file mode 100644
index 0000000..8e841f6
--- /dev/null
+++ b/database/storages.go
@@ -0,0 +1,36 @@
+package database
+
+// A Factory creates a new database of it's type.
+type Factory func(name, location string) (*storage.Interface, error)
+
+var (
+  storages map[string]Factory
+  storagesLock sync.Mutex
+)
+
+// RegisterStorage registers a new storage type.
+func RegisterStorage(name string, factory Factory) error {
+  storagesLock.Lock()
+  defer storagesLock.Unlock()
+
+  _, ok := storages[name]
+  if ok {
+    return errors.New("factory for this type already exists")
+  }
+
+  storages[name] = factory
+  return nil
+}
+
+// startDatabase starts a new database with the given name, storageType at location.
+func startDatabase(name, storageType, location string) (*storage.Interface, error) {
+  storagesLock.Lock()
+  defer storagesLock.Unlock()
+
+  factory, ok := storages[name]
+  if !ok {
+    return fmt.Errorf("storage of this type (%s) does not exist", storageType)
+  }
+
+  return factory(name, location)
+}

From 2716b9663c3586c482b3b63d710ce01602d7deed Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 4 Sep 2018 17:10:11 +0200
Subject: [PATCH 07/20] Remove BSON from DSD for now, add GenCode ID

---
 formats/dsd/dsd.go | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/formats/dsd/dsd.go b/formats/dsd/dsd.go
index c21f75f..7b3fcd2 100644
--- a/formats/dsd/dsd.go
+++ b/formats/dsd/dsd.go
@@ -10,19 +10,19 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/pkg/bson"
+	// "github.com/pkg/bson"
 
 	"github.com/Safing/safing-core/formats/varint"
 )
 
 // define types
 const (
-	AUTO   = 0
-	STRING = 83 // S
-	BYTES  = 88 // X
-	JSON   = 74 // J
-	BSON   = 66 // B
-	// MSGP
+	AUTO    = 0
+	STRING  = 83 // S
+	BYTES   = 88 // X
+	JSON    = 74 // J
+	BSON    = 66 // B
+	GenCode = 71 // G (reserved)
 )
 
 // define errors
@@ -56,12 +56,12 @@ func Load(data []byte, t interface{}) (interface{}, error) {
 			return nil, err
 		}
 		return t, nil
-	case BSON:
-		err := bson.Unmarshal(data[read:], t)
-		if err != nil {
-			return nil, err
-		}
-		return t, nil
+	// case BSON:
+	// 	err := bson.Unmarshal(data[read:], t)
+	// 	if err != nil {
+	// 		return nil, err
+	// 	}
+	// 	return t, nil
 	// case MSGP:
 	//   err := t.UnmarshalMsg(data[read:])
 	//   if err != nil {
@@ -101,11 +101,11 @@ func Dump(t interface{}, format uint8) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
-	case BSON:
-		data, err = bson.Marshal(t)
-		if err != nil {
-			return nil, err
-		}
+	// case BSON:
+	// 	data, err = bson.Marshal(t)
+	// 	if err != nil {
+	// 		return nil, err
+	// 	}
 	// case MSGP:
 	//   data, err := t.MarshalMsg(nil)
 	//   if err != nil {

From 94598b115b428e0b9aaa33941e87cbf00f1356b9 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 5 Sep 2018 17:05:23 +0200
Subject: [PATCH 08/20] Finish badger and sinkhole storage

---
 database/storage/badger/badger.go     | 145 ++++++++++++++++++++++++++
 database/storage/bbolt/bbolt.go       |   9 --
 database/storage/errors.go            |   8 ++
 database/storage/interface.go         |  24 +----
 database/storage/sinkhole/sinkhole.go |  66 ++++++++++++
 database/storage/storages.go          |  42 ++++++++
 6 files changed, 266 insertions(+), 28 deletions(-)
 delete mode 100644 database/storage/bbolt/bbolt.go
 create mode 100644 database/storage/errors.go
 create mode 100644 database/storage/sinkhole/sinkhole.go
 create mode 100644 database/storage/storages.go

diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
index 2b90f77..83aa821 100644
--- a/database/storage/badger/badger.go
+++ b/database/storage/badger/badger.go
@@ -1 +1,146 @@
 package badger
+
+import (
+	"errors"
+	"time"
+
+	"github.com/dgraph-io/badger"
+
+	"github.com/Safing/portbase/database/iterator"
+	"github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/storage"
+)
+
+// Badger database made pluggable for portbase.
+type Badger struct {
+	name string
+	db   *badger.DB
+}
+
+func init() {
+	storage.Register("badger", NewBadger)
+}
+
+// NewBadger opens/creates a badger database.
+func NewBadger(name, location string) (storage.Interface, error) {
+	opts := badger.DefaultOptions
+	opts.Dir = location
+	opts.ValueDir = location
+
+	db, err := badger.Open(opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Badger{
+		name: name,
+		db:   db,
+	}, nil
+}
+
+// Exists returns whether an entry with the given key exists.
+func (b *Badger) Exists(key string) (bool, error) {
+	err := b.db.View(func(txn *badger.Txn) error {
+		_, err := txn.Get([]byte(key))
+		if err != nil {
+			if err == badger.ErrKeyNotFound {
+				return nil
+			}
+			return err
+		}
+		return nil
+	})
+	if err == nil {
+		return true, nil
+	}
+	return false, nil
+}
+
+// Get returns a database record.
+func (b *Badger) Get(key string) (model.Model, error) {
+	var item *badger.Item
+
+	err := b.db.View(func(txn *badger.Txn) error {
+		var err error
+		item, err = txn.Get([]byte(key))
+		if err != nil {
+			if err == badger.ErrKeyNotFound {
+				return storage.ErrNotFound
+			}
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if item.IsDeletedOrExpired() {
+		return nil, storage.ErrNotFound
+	}
+
+	data, err := item.ValueCopy(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := model.NewRawWrapper(b.name, string(item.Key()), data)
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// Put stores a record in the database.
+func (b *Badger) Put(m model.Model) error {
+	data, err := m.MarshalRecord()
+	if err != nil {
+		return err
+	}
+
+	err = b.db.Update(func(txn *badger.Txn) error {
+		if m.Meta().GetAbsoluteExpiry() > 0 {
+			txn.SetWithTTL([]byte(m.DatabaseKey()), data, time.Duration(m.Meta().GetRelativeExpiry()))
+		} else {
+			txn.Set([]byte(m.DatabaseKey()), data)
+		}
+		return nil
+	})
+	return err
+}
+
+// Delete deletes a record from the database.
+func (b *Badger) Delete(key string) error {
+	return b.db.Update(func(txn *badger.Txn) error {
+		err := txn.Delete([]byte(key))
+		if err != nil && err != badger.ErrKeyNotFound {
+			return err
+		}
+		return nil
+	})
+}
+
+// Query returns a an iterator for the supplied query.
+func (b *Badger) Query(q *query.Query) (*iterator.Iterator, error) {
+	return nil, errors.New("query not implemented by badger")
+}
+
+// Maintain runs a light maintenance operation on the database.
+func (b *Badger) Maintain() error {
+	b.db.RunValueLogGC(0.7)
+	return nil
+}
+
+// MaintainThorough runs a thorough maintenance operation on the database.
+func (b *Badger) MaintainThorough() (err error) {
+	for err == nil {
+		err = b.db.RunValueLogGC(0.7)
+	}
+	return nil
+}
+
+// Shutdown shuts down the database.
+func (b *Badger) Shutdown() error {
+	return b.db.Close()
+}
diff --git a/database/storage/bbolt/bbolt.go b/database/storage/bbolt/bbolt.go
deleted file mode 100644
index 0c18c51..0000000
--- a/database/storage/bbolt/bbolt.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package bbolt
-
-import bolt "go.etcd.io/bbolt"
-
-db, err := bolt.Open(path, 0666, nil)
-if err != nil {
-  return err
-}
-defer db.Close()
diff --git a/database/storage/errors.go b/database/storage/errors.go
new file mode 100644
index 0000000..91c7689
--- /dev/null
+++ b/database/storage/errors.go
@@ -0,0 +1,8 @@
+package storage
+
+import "errors"
+
+// Errors for storages
+var (
+	ErrNotFound = errors.New("not found")
+)
diff --git a/database/storage/interface.go b/database/storage/interface.go
index e1fa952..f6753f9 100644
--- a/database/storage/interface.go
+++ b/database/storage/interface.go
@@ -8,27 +8,13 @@ import (
 
 // Interface defines the database storage API.
 type Interface interface {
-	// Retrieve
 	Exists(key string) (bool, error)
 	Get(key string) (model.Model, error)
-
-	// Modify
-	Create(model model.Model) error
-	Update(model model.Model) error         // create when not exists
-	UpdateOrCreate(model model.Model) error // update, create if not exists.
+	Put(m model.Model) error
 	Delete(key string) error
+	Query(q *query.Query) (*iterator.Iterator, error)
 
-	// Partial
-	// What happens if I mutate a value that does not yet exist? How would I know its type?
-	InsertPartial(key string, partialObject interface{})
-	InsertValue(key string, attribute string, value interface{})
-
-	// Query
-	Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
-
-	// Meta
-	SetAbsoluteExpiry(key string, time int64)
-	SetRelativateExpiry(key string, duration int64)
-	MakeCrownJewel(key string)
-	MakeSecret(key string)
+	Maintain() error
+	MaintainThorough() error
+	Shutdown() error
 }
diff --git a/database/storage/sinkhole/sinkhole.go b/database/storage/sinkhole/sinkhole.go
new file mode 100644
index 0000000..fe6ec4f
--- /dev/null
+++ b/database/storage/sinkhole/sinkhole.go
@@ -0,0 +1,66 @@
+package sinkhole
+
+import (
+	"errors"
+
+	"github.com/Safing/portbase/database/iterator"
+	"github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/storage"
+)
+
+// Sinkhole is a dummy storage.
+type Sinkhole struct {
+	name string
+}
+
+func init() {
+	storage.Register("sinkhole", NewSinkhole)
+}
+
+// NewSinkhole creates a dummy database.
+func NewSinkhole(name, location string) (storage.Interface, error) {
+	return &Sinkhole{
+		name: name,
+	}, nil
+}
+
+// Exists returns whether an entry with the given key exists.
+func (s *Sinkhole) Exists(key string) (bool, error) {
+	return false, nil
+}
+
+// Get returns a database record.
+func (s *Sinkhole) Get(key string) (model.Model, error) {
+	return nil, storage.ErrNotFound
+}
+
+// Put stores a record in the database.
+func (s *Sinkhole) Put(m model.Model) error {
+	return nil
+}
+
+// Delete deletes a record from the database.
+func (s *Sinkhole) Delete(key string) error {
+	return nil
+}
+
+// Query returns a an iterator for the supplied query.
+func (s *Sinkhole) Query(q *query.Query) (*iterator.Iterator, error) {
+	return nil, errors.New("query not implemented by sinkhole")
+}
+
+// Maintain runs a light maintenance operation on the database.
+func (s *Sinkhole) Maintain() error {
+	return nil
+}
+
+// MaintainThorough runs a thorough maintenance operation on the database.
+func (s *Sinkhole) MaintainThorough() (err error) {
+	return nil
+}
+
+// Shutdown shuts down the database.
+func (s *Sinkhole) Shutdown() error {
+	return nil
+}
diff --git a/database/storage/storages.go b/database/storage/storages.go
new file mode 100644
index 0000000..f4a3c27
--- /dev/null
+++ b/database/storage/storages.go
@@ -0,0 +1,42 @@
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// A Factory creates a new database of it's type.
+type Factory func(name, location string) (Interface, error)
+
+var (
+	storages     map[string]Factory
+	storagesLock sync.Mutex
+)
+
+// Register registers a new storage type.
+func Register(name string, factory Factory) error {
+	storagesLock.Lock()
+	defer storagesLock.Unlock()
+
+	_, ok := storages[name]
+	if ok {
+		return errors.New("factory for this type already exists")
+	}
+
+	storages[name] = factory
+	return nil
+}
+
+// StartDatabase starts a new database with the given name and storageType at location.
+func StartDatabase(name, storageType, location string) (Interface, error) {
+	storagesLock.Lock()
+	defer storagesLock.Unlock()
+
+	factory, ok := storages[name]
+	if !ok {
+		return nil, fmt.Errorf("storage of this type (%s) does not exist", storageType)
+	}
+
+	return factory(name, location)
+}

From 9b7365376ccaef5810bf82908091759965ec7c19 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 5 Sep 2018 17:05:51 +0200
Subject: [PATCH 09/20] Finalize model

---
 database/controller.go         |  38 ++++++++++
 database/databases.go          |   4 +-
 database/interface.go          |  20 +++--
 database/model/base.go         |  44 ++++++++++-
 database/model/base_test.go    |  13 ++++
 database/model/key.go          |   8 +-
 database/model/meta-gencode.go |   9 ++-
 database/model/meta.go         |  32 ++++++--
 database/model/model.go        |  14 +++-
 database/model/model_test.go   |  16 ++++
 database/model/wrapper.go      | 131 +++++++++++++++++++++++----------
 database/model/wrapper_test.go |  55 ++++++++++++++
 database/storages.go           |  36 ---------
 13 files changed, 320 insertions(+), 100 deletions(-)
 create mode 100644 database/controller.go
 create mode 100644 database/model/base_test.go
 create mode 100644 database/model/model_test.go
 create mode 100644 database/model/wrapper_test.go
 delete mode 100644 database/storages.go

diff --git a/database/controller.go b/database/controller.go
new file mode 100644
index 0000000..42fcfbe
--- /dev/null
+++ b/database/controller.go
@@ -0,0 +1,38 @@
+package database
+
+type Controller struct {
+  storage
+  writeLock sync.RWMutex
+  readLock sync.RWMutex
+  migrating *abool.AtomicBool
+}
+
+func NewController() (*Controller, error) {
+
+}
+
+	// Retrieve
+func (c *Controller) Exists(key string) (bool, error) {}
+func (c *Controller) Get(key string) (model.Model, error) {}
+
+// Modify
+func (c *Controller) Create(model model.Model) error {}
+// create when not exists
+func (c *Controller) Update(model model.Model) error {}
+// update, create if not exists.
+func (c *Controller) UpdateOrCreate(model model.Model) error {}
+func (c *Controller) Delete(key string) error {}
+
+// Partial
+// What happens if I mutate a value that does not yet exist? How would I know its type?
+func (c *Controller) InsertPartial(key string, partialObject interface{}) {}
+func (c *Controller) InsertValue(key string, attribute string, value interface{}) {}
+
+// Query
+func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {}
+
+// Meta
+func (c *Controller) SetAbsoluteExpiry(key string, time int64) {}
+func (c *Controller) SetRelativateExpiry(key string, duration int64) {}
+func (c *Controller) MakeCrownJewel(key string) {}
+func (c *Controller) MakeSecret(key string) {}
diff --git a/database/databases.go b/database/databases.go
index d4643f5..b9cc0e9 100644
--- a/database/databases.go
+++ b/database/databases.go
@@ -2,11 +2,11 @@ package database
 
 
 var (
-  databases = make(map[string]*storage.Interface)
+  databases = make(map[string]*Controller)
   databasesLock sync.Mutex
 )
 
-func getDatabase(name string) *storage.Interface {
+func getDatabase(name string) *Controller {
   databasesLock.Lock()
   defer   databasesLock.Unlock()
   storage, ok := databases[name]
diff --git a/database/interface.go b/database/interface.go
index 2d249ba..7d5ed54 100644
--- a/database/interface.go
+++ b/database/interface.go
@@ -1,17 +1,27 @@
 package database
 
-type Interface struct {
-  local bool
-  internal bool
+// Interface provides a method to access the database with attached options.
+type Interface struct {}
+
+// Options holds options that may be set for an Interface instance.
+type Options struct {
+  Local bool
+  Internal bool
+  AlwaysMakeSecret bool
+  AlwaysMakeCrownjewel bool
 }
 
-func NewInterface(local bool, internal bool) *Interface {
+// NewInterface returns a new Interface to the database.
+func NewInterface(opts *Options) *Interface {
   return &Interface{
     local: local,
     internal: internal,
   }
 }
 
-func (i *Interface) Get(string key) (model.Model, error) {
+func (i *Interface) Get(key string) (model.Model, error) {
+
+  controller 
+
   return nil, nil
 }
diff --git a/database/model/base.go b/database/model/base.go
index 5ad252d..03ed22a 100644
--- a/database/model/base.go
+++ b/database/model/base.go
@@ -1,6 +1,10 @@
 package model
 
 import (
+	"errors"
+	"fmt"
+
+	"github.com/Safing/portbase/container"
 	"github.com/Safing/portbase/formats/dsd"
 )
 
@@ -13,17 +17,27 @@ type Base struct {
 
 // Key returns the key of the database record.
 func (b *Base) Key() string {
+	return fmt.Sprintf("%s:%s", b.dbName, b.dbKey)
+}
+
+// DatabaseName returns the name of the database.
+func (b *Base) DatabaseName() string {
+	return b.dbName
+}
+
+// DatabaseKey returns the database key of the database record.
+func (b *Base) DatabaseKey() string {
 	return b.dbKey
 }
 
 // SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
 func (b *Base) SetKey(key string) {
-	b.dbKey = key
+	b.dbName, b.dbKey = ParseKey(key)
 }
 
 // MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
 func (b *Base) MoveTo(key string) {
-	b.dbKey = key
+	b.SetKey(key)
 	b.meta.Reset()
 }
 
@@ -45,3 +59,29 @@ func (b *Base) Marshal(format uint8) ([]byte, error) {
 	}
 	return dumped, nil
 }
+
+// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+func (b *Base) MarshalRecord() ([]byte, error) {
+	if b.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	// version
+	c := container.New([]byte{1})
+
+	// meta
+	metaSection, err := b.meta.GenCodeMarshal(nil)
+	if err != nil {
+		return nil, err
+	}
+	c.AppendAsBlock(metaSection)
+
+	// data
+	dataSection, err := b.Marshal(dsd.JSON)
+	if err != nil {
+		return nil, err
+	}
+	c.Append(dataSection)
+
+	return c.CompileData(), nil
+}
diff --git a/database/model/base_test.go b/database/model/base_test.go
new file mode 100644
index 0000000..a1fbe84
--- /dev/null
+++ b/database/model/base_test.go
@@ -0,0 +1,13 @@
+package model
+
+import "testing"
+
+func TestBaseModel(t *testing.T) {
+
+	// check model interface compliance
+	var m Model
+	b := &TestModel{}
+	m = b
+	_ = m
+
+}
diff --git a/database/model/key.go b/database/model/key.go
index 19c7c59..81dee11 100644
--- a/database/model/key.go
+++ b/database/model/key.go
@@ -1,14 +1,14 @@
 package model
 
 import (
-	"errors"
 	"strings"
 )
 
-func ParseKey(key string) (dbName, dbKey string, err error) {
+// ParseKey splits a key into it's database name and key parts.
+func ParseKey(key string) (dbName, dbKey string) {
 	splitted := strings.SplitN(key, ":", 2)
 	if len(splitted) == 2 {
-		return splitted[0], splitted[1], nil
+		return splitted[0], splitted[1]
 	}
-	return "", "", errors.New("invalid key")
+	return splitted[0], ""
 }
diff --git a/database/model/meta-gencode.go b/database/model/meta-gencode.go
index 6c4e76f..7c3f494 100644
--- a/database/model/meta-gencode.go
+++ b/database/model/meta-gencode.go
@@ -1,6 +1,7 @@
 package model
 
 import (
+	"fmt"
 	"io"
 	"time"
 	"unsafe"
@@ -13,7 +14,7 @@ var (
 )
 
 // GenCodeSize returns the size of the gencode marshalled byte slice
-func (d *Meta) GenCodeSize() (s uint64) {
+func (d *Meta) GenCodeSize() (s int) {
 	s += 34
 	return
 }
@@ -22,7 +23,7 @@ func (d *Meta) GenCodeSize() (s uint64) {
 func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
 	size := d.GenCodeSize()
 	{
-		if uint64(cap(buf)) >= size {
+		if cap(buf) >= size {
 			buf = buf[:size]
 		} else {
 			buf = make([]byte, size)
@@ -125,6 +126,10 @@ func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
 
 // GenCodeUnmarshal gencode unmarshalls Meta and returns the bytes read.
 func (d *Meta) GenCodeUnmarshal(buf []byte) (uint64, error) {
+	if len(buf) < d.GenCodeSize() {
+		return 0, fmt.Errorf("insufficient data: got %d out of %d bytes", len(buf), d.GenCodeSize())
+	}
+
 	i := uint64(0)
 
 	{
diff --git a/database/model/meta.go b/database/model/meta.go
index 2424821..057a8a4 100644
--- a/database/model/meta.go
+++ b/database/model/meta.go
@@ -12,19 +12,37 @@ type Meta struct {
 	cronjewel bool // crownjewels must never leave the instance, but may be read by the UI
 }
 
-// SetAbsoluteExpiry sets an absolute expiry time, that is not affected when the record is updated.
-func (m *Meta) SetAbsoluteExpiry(time int64) {
-	m.expires = time
+// SetAbsoluteExpiry sets an absolute expiry time (in seconds), that is not affected when the record is updated.
+func (m *Meta) SetAbsoluteExpiry(seconds int64) {
+	m.expires = seconds
 	m.deleted = 0
 }
 
-// SetRelativateExpiry sets a relative expiry that is automatically updated whenever the record is updated/saved.
-func (m *Meta) SetRelativateExpiry(duration int64) {
-	if duration >= 0 {
-		m.deleted = -duration
+// SetRelativateExpiry sets a relative expiry time (ie. TTL in seconds) that is automatically updated whenever the record is updated/saved.
+func (m *Meta) SetRelativateExpiry(seconds int64) {
+	if seconds >= 0 {
+		m.deleted = -seconds
 	}
 }
 
+// GetAbsoluteExpiry returns the absolute expiry time.
+func (m *Meta) GetAbsoluteExpiry() int64 {
+	return m.expires
+}
+
+// GetRelativeExpiry returns the current relative expiry time - ie. seconds until expiry.
+func (m *Meta) GetRelativeExpiry() int64 {
+	if m.deleted < 0 {
+		return -m.deleted
+	}
+
+	abs := m.expires - time.Now().Unix()
+	if abs < 0 {
+		return 0
+	}
+	return abs
+}
+
 // MakeCrownJewel marks the database records as a crownjewel, meaning that it will not be sent/synced to other devices.
 func (m *Meta) MakeCrownJewel() {
 	m.cronjewel = true
diff --git a/database/model/model.go b/database/model/model.go
index 951f6f3..71fe032 100644
--- a/database/model/model.go
+++ b/database/model/model.go
@@ -2,10 +2,18 @@ package model
 
 // Model provides an interface for uniformally handling database records.
 type Model interface {
-	Key() string
-	SetKey(key string)
-	MoveTo(key string)
+	Key() string          // test:config
+	DatabaseName() string // test
+	DatabaseKey() string  // config
+
+	SetKey(key string) // test:config
+	MoveTo(key string) // test:config
 	Meta() *Meta
 	SetMeta(meta *Meta)
+
 	Marshal(format uint8) ([]byte, error)
+	MarshalRecord() ([]byte, error)
+
+	Lock()
+	Unlock()
 }
diff --git a/database/model/model_test.go b/database/model/model_test.go
new file mode 100644
index 0000000..de732f7
--- /dev/null
+++ b/database/model/model_test.go
@@ -0,0 +1,16 @@
+package model
+
+import "sync"
+
+type TestModel struct {
+	Base
+	lock sync.Mutex
+}
+
+func (tm *TestModel) Lock() {
+	tm.lock.Lock()
+}
+
+func (tm *TestModel) Unlock() {
+	tm.lock.Unlock()
+}
diff --git a/database/model/wrapper.go b/database/model/wrapper.go
index a19f9d1..bdb9c2a 100644
--- a/database/model/wrapper.go
+++ b/database/model/wrapper.go
@@ -3,59 +3,74 @@ package model
 import (
 	"errors"
 	"fmt"
+	"sync"
 
-	"github.com/Safing/safing-core/formats/dsd"
-	"github.com/Safing/safing-core/formats/varint"
+	"github.com/Safing/portbase/container"
+	"github.com/Safing/portbase/formats/dsd"
+	"github.com/Safing/portbase/formats/varint"
 )
 
 type Wrapper struct {
-	dbName string
-	dbKey  string
-	meta   *Meta
+	Base
 	Format uint8
 	Data   []byte
+	lock   sync.Mutex
 }
 
+func NewRawWrapper(database, key string, data []byte) (*Wrapper, error) {
+	version, offset, err := varint.Unpack8(data)
+	if version != 1 {
+		return nil, fmt.Errorf("incompatible record version: %d", version)
+	}
+
+	metaSection, n, err := varint.GetNextBlock(data[offset:])
+	if err != nil {
+		return nil, fmt.Errorf("could not get meta section: %s", err)
+	}
+	offset += n
+
+	newMeta := &Meta{}
+	_, err = newMeta.GenCodeUnmarshal(metaSection)
+	if err != nil {
+		return nil, fmt.Errorf("could not unmarshal meta section: %s", err)
+	}
+
+	format, _, err := varint.Unpack8(data[offset:])
+	if err != nil {
+		return nil, fmt.Errorf("could not get dsd format: %s", err)
+	}
+
+	return &Wrapper{
+		Base{
+			database,
+			key,
+			newMeta,
+		},
+		format,
+		data[offset:],
+		sync.Mutex{},
+	}, nil
+}
+
+// NewWrapper returns a new model wrapper for the given data.
 func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
 	format, _, err := varint.Unpack8(data)
 	if err != nil {
-		return nil, fmt.Errorf("database: could not get dsd format: %s", err)
+		return nil, fmt.Errorf("could not get dsd format: %s", err)
 	}
 
-	new := &Wrapper{
-		dbKey:  key,
-		meta:   meta,
-		Format: format,
-		Data:   data,
-	}
+	dbName, dbKey := ParseKey(key)
 
-	return new, nil
-}
-
-// Key returns the key of the database record.
-func (w *Wrapper) Key() string {
-	return w.dbKey
-}
-
-// SetKey sets the key on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
-func (w *Wrapper) SetKey(key string) {
-	w.dbKey = key
-}
-
-// MoveTo sets a new key for the record and resets all metadata, except for the secret and crownjewel status.
-func (w *Wrapper) MoveTo(key string) {
-	w.dbKey = key
-	w.meta.Reset()
-}
-
-// Meta returns the metadata object for this record.
-func (w *Wrapper) Meta() *Meta {
-	return w.meta
-}
-
-// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
-func (w *Wrapper) SetMeta(meta *Meta) {
-	w.meta = meta
+	return &Wrapper{
+		Base{
+			dbName: dbName,
+			dbKey:  dbKey,
+			meta:   meta,
+		},
+		format,
+		data,
+		sync.Mutex{},
+	}, nil
 }
 
 // Marshal marshals the object, without the database key or metadata
@@ -65,3 +80,41 @@ func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
 	}
 	return w.Data, nil
 }
+
+// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+func (w *Wrapper) MarshalRecord() ([]byte, error) {
+	// Duplication necessary, as the version from Base would call Base.Marshal instead of Wrapper.Marshal
+
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	// version
+	c := container.New([]byte{1})
+
+	// meta
+	metaSection, err := w.meta.GenCodeMarshal(nil)
+	if err != nil {
+		return nil, err
+	}
+	c.AppendAsBlock(metaSection)
+
+	// data
+	dataSection, err := w.Marshal(dsd.JSON)
+	if err != nil {
+		return nil, err
+	}
+	c.Append(dataSection)
+
+	return c.CompileData(), nil
+}
+
+// Lock locks the record.
+func (w *Wrapper) Lock() {
+	w.lock.Lock()
+}
+
+// Unlock unlocks the record.
+func (w *Wrapper) Unlock() {
+	w.lock.Unlock()
+}
diff --git a/database/model/wrapper_test.go b/database/model/wrapper_test.go
new file mode 100644
index 0000000..f1c7c4a
--- /dev/null
+++ b/database/model/wrapper_test.go
@@ -0,0 +1,55 @@
+package model
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/Safing/portbase/formats/dsd"
+)
+
+func TestWrapper(t *testing.T) {
+
+	// check model interface compliance
+	var m Model
+	w := &Wrapper{}
+	m = w
+	_ = m
+
+	// create test data
+	testData := []byte(`J{"a": "b"}`)
+
+	// test wrapper
+	wrapper, err := NewWrapper("test:a", nil, testData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if wrapper.Format != dsd.JSON {
+		t.Error("format mismatch")
+	}
+	if !bytes.Equal(testData, wrapper.Data) {
+		t.Error("data mismatch")
+	}
+
+	encoded, err := wrapper.Marshal(dsd.JSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(testData, encoded) {
+		t.Error("marshal mismatch")
+	}
+
+	wrapper.SetMeta(&Meta{})
+	raw, err := wrapper.MarshalRecord()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wrapper2, err := NewRawWrapper("test", "a", raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(testData, wrapper2.Data) {
+		t.Error("marshal mismatch")
+	}
+
+}
diff --git a/database/storages.go b/database/storages.go
deleted file mode 100644
index 8e841f6..0000000
--- a/database/storages.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package database
-
-// A Factory creates a new database of it's type.
-type Factory func(name, location string) (*storage.Interface, error)
-
-var (
-  storages map[string]Factory
-  storagesLock sync.Mutex
-)
-
-// RegisterStorage registers a new storage type.
-func RegisterStorage(name string, factory Factory) error {
-  storagesLock.Lock()
-  defer storagesLock.Unlock()
-
-  _, ok := storages[name]
-  if ok {
-    return errors.New("factory for this type already exists")
-  }
-
-  storages[name] = factory
-  return nil
-}
-
-// startDatabase starts a new database with the given name, storageType at location.
-func startDatabase(name, storageType, location string) (*storage.Interface, error) {
-  storagesLock.Lock()
-  defer storagesLock.Unlock()
-
-  factory, ok := storages[name]
-  if !ok {
-    return fmt.Errorf("storage of this type (%s) does not exist", storageType)
-  }
-
-  return factory(name, location)
-}

From d76bfd55dc1f9822a21a251089673e3c9350a303 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 5 Sep 2018 17:19:45 +0200
Subject: [PATCH 10/20] Rename model.Model to record.Record

---
 database/controller.go                        |  12 +-
 database/interface.go                         |   4 +-
 database/iterator/iterator.go                 |   6 +-
 database/model.go                             |  24 -
 database/model/model/Colfer.go                | 507 ------------------
 database/namespaces.go                        |  61 ---
 database/{model => record}/base.go            |   0
 database/{model => record}/base_test.go       |   6 +-
 database/{model => record}/formats.go         |   0
 database/{model => record}/key.go             |   0
 database/{model => record}/meta-bench_test.go |   0
 database/{model => record}/meta-gencode.go    |   0
 .../{model => record}/meta-gencode_test.go    |   0
 database/{model => record}/meta.colf          |   0
 database/{model => record}/meta.gencode       |   0
 database/{model => record}/meta.go            |   0
 database/{model/model.go => record/record.go} |   4 +-
 .../model_test.go => record/record_test.go}   |   6 +-
 database/{model => record}/wrapper.go         |   0
 database/{model => record}/wrapper_test.go    |   2 +-
 database/storage/badger/badger.go             |   6 +-
 database/storage/interface.go                 |   6 +-
 database/storage/sinkhole/sinkhole.go         |   6 +-
 23 files changed, 31 insertions(+), 619 deletions(-)
 delete mode 100644 database/model/model/Colfer.go
 delete mode 100644 database/namespaces.go
 rename database/{model => record}/base.go (100%)
 rename database/{model => record}/base_test.go (55%)
 rename database/{model => record}/formats.go (100%)
 rename database/{model => record}/key.go (100%)
 rename database/{model => record}/meta-bench_test.go (100%)
 rename database/{model => record}/meta-gencode.go (100%)
 rename database/{model => record}/meta-gencode_test.go (100%)
 rename database/{model => record}/meta.colf (100%)
 rename database/{model => record}/meta.gencode (100%)
 rename database/{model => record}/meta.go (100%)
 rename database/{model/model.go => record/record.go} (76%)
 rename database/{model/model_test.go => record/record_test.go} (51%)
 rename database/{model => record}/wrapper.go (100%)
 rename database/{model => record}/wrapper_test.go (98%)

diff --git a/database/controller.go b/database/controller.go
index 42fcfbe..499b746 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -1,5 +1,9 @@
 package database
 
+import (
+  "github.com/Safing/portbase/database/record"
+)
+
 type Controller struct {
   storage
   writeLock sync.RWMutex
@@ -13,14 +17,14 @@ func NewController() (*Controller, error) {
 
 	// Retrieve
 func (c *Controller) Exists(key string) (bool, error) {}
-func (c *Controller) Get(key string) (model.Model, error) {}
+func (c *Controller) Get(key string) (record.Record, error) {}
 
 // Modify
-func (c *Controller) Create(model model.Model) error {}
+func (c *Controller) Create(model record.Record) error {}
 // create when not exists
-func (c *Controller) Update(model model.Model) error {}
+func (c *Controller) Update(model record.Record) error {}
 // update, create if not exists.
-func (c *Controller) UpdateOrCreate(model model.Model) error {}
+func (c *Controller) UpdateOrCreate(model record.Record) error {}
 func (c *Controller) Delete(key string) error {}
 
 // Partial
diff --git a/database/interface.go b/database/interface.go
index 7d5ed54..dd56955 100644
--- a/database/interface.go
+++ b/database/interface.go
@@ -19,9 +19,9 @@ func NewInterface(opts *Options) *Interface {
   }
 }
 
-func (i *Interface) Get(key string) (model.Model, error) {
+func (i *Interface) Get(key string) (record.Record, error) {
 
-  controller 
+  controller
 
   return nil, nil
 }
diff --git a/database/iterator/iterator.go b/database/iterator/iterator.go
index e045d6b..210731a 100644
--- a/database/iterator/iterator.go
+++ b/database/iterator/iterator.go
@@ -1,18 +1,18 @@
 package iterator
 
 import (
-	"github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/record"
 )
 
 // Iterator defines the iterator structure.
 type Iterator struct {
-	Next  chan model.Model
+	Next  chan record.Record
 	Error error
 }
 
 // New creates a new Iterator.
 func New() *Iterator {
 	return &Iterator{
-		Next: make(chan model.Model, 10),
+		Next: make(chan record.Record, 10),
 	}
 }
diff --git a/database/model.go b/database/model.go
index 2644043..1aa7634 100644
--- a/database/model.go
+++ b/database/model.go
@@ -4,36 +4,12 @@ package database
 
 import (
 	"fmt"
-	"strings"
 	"sync"
 
 	"github.com/Safing/safing-core/database/dbutils"
 	"github.com/Safing/safing-core/formats/dsd"
 )
 
-type Model interface {
-	SetKey(*datastore.Key)
-	GetKey() *datastore.Key
-	FmtKey() string
-	// Type() string
-	// DefaultNamespace() datastore.Key
-	// Create(string) error
-	// CreateInLocation(datastore.Key, string) error
-	// CreateObject(*datastore.Key, string, Model) error
-	// Save() error
-	// Delete() error
-	// CastError(interface{}, interface{}) error
-}
-
-func getTypeName(model interface{}) string {
-	full := fmt.Sprintf("%T", model)
-	return full[strings.LastIndex(full, ".")+1:]
-}
-
-func TypeAssertError(model Model, object interface{}) error {
-	return fmt.Errorf("database: could not assert %s to type %T (is type %T)", model.FmtKey(), model, object)
-}
-
 // Model Registration
 
 var (
diff --git a/database/model/model/Colfer.go b/database/model/model/Colfer.go
deleted file mode 100644
index 0f68a19..0000000
--- a/database/model/model/Colfer.go
+++ /dev/null
@@ -1,507 +0,0 @@
-package model
-
-// Code generated by colf(1); DO NOT EDIT.
-// The compiler used schema file meta.colf.
-
-import (
-	"encoding/binary"
-	"fmt"
-	"io"
-)
-
-var intconv = binary.BigEndian
-
-// Colfer configuration attributes
-var (
-	// ColferSizeMax is the upper limit for serial byte sizes.
-	ColferSizeMax = 16 * 1024 * 1024
-)
-
-// ColferMax signals an upper limit breach.
-type ColferMax string
-
-// Error honors the error interface.
-func (m ColferMax) Error() string { return string(m) }
-
-// ColferError signals a data mismatch as as a byte index.
-type ColferError int
-
-// Error honors the error interface.
-func (i ColferError) Error() string {
-	return fmt.Sprintf("colfer: unknown header at byte %d", i)
-}
-
-// ColferTail signals data continuation as a byte index.
-type ColferTail int
-
-// Error honors the error interface.
-func (i ColferTail) Error() string {
-	return fmt.Sprintf("colfer: data continuation at byte %d", i)
-}
-
-type Course struct {
-	Created int64
-
-	Modified int64
-
-	Expires int64
-
-	Deleted int64
-
-	Secret bool
-
-	Cronjewel bool
-}
-
-// MarshalTo encodes o as Colfer into buf and returns the number of bytes written.
-// If the buffer is too small, MarshalTo will panic.
-func (o *Course) MarshalTo(buf []byte) int {
-	var i int
-
-	if v := o.Created; v != 0 {
-		x := uint64(v)
-		if v >= 0 {
-			buf[i] = 0
-		} else {
-			x = ^x + 1
-			buf[i] = 0 | 0x80
-		}
-		i++
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			buf[i] = byte(x | 0x80)
-			x >>= 7
-			i++
-		}
-		buf[i] = byte(x)
-		i++
-	}
-
-	if v := o.Modified; v != 0 {
-		x := uint64(v)
-		if v >= 0 {
-			buf[i] = 1
-		} else {
-			x = ^x + 1
-			buf[i] = 1 | 0x80
-		}
-		i++
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			buf[i] = byte(x | 0x80)
-			x >>= 7
-			i++
-		}
-		buf[i] = byte(x)
-		i++
-	}
-
-	if v := o.Expires; v != 0 {
-		x := uint64(v)
-		if v >= 0 {
-			buf[i] = 2
-		} else {
-			x = ^x + 1
-			buf[i] = 2 | 0x80
-		}
-		i++
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			buf[i] = byte(x | 0x80)
-			x >>= 7
-			i++
-		}
-		buf[i] = byte(x)
-		i++
-	}
-
-	if v := o.Deleted; v != 0 {
-		x := uint64(v)
-		if v >= 0 {
-			buf[i] = 3
-		} else {
-			x = ^x + 1
-			buf[i] = 3 | 0x80
-		}
-		i++
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			buf[i] = byte(x | 0x80)
-			x >>= 7
-			i++
-		}
-		buf[i] = byte(x)
-		i++
-	}
-
-	if o.Secret {
-		buf[i] = 4
-		i++
-	}
-
-	if o.Cronjewel {
-		buf[i] = 5
-		i++
-	}
-
-	buf[i] = 0x7f
-	i++
-	return i
-}
-
-// MarshalLen returns the Colfer serial byte size.
-// The error return option is model.ColferMax.
-func (o *Course) MarshalLen() (int, error) {
-	l := 1
-
-	if v := o.Created; v != 0 {
-		l += 2
-		x := uint64(v)
-		if v < 0 {
-			x = ^x + 1
-		}
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			x >>= 7
-			l++
-		}
-	}
-
-	if v := o.Modified; v != 0 {
-		l += 2
-		x := uint64(v)
-		if v < 0 {
-			x = ^x + 1
-		}
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			x >>= 7
-			l++
-		}
-	}
-
-	if v := o.Expires; v != 0 {
-		l += 2
-		x := uint64(v)
-		if v < 0 {
-			x = ^x + 1
-		}
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			x >>= 7
-			l++
-		}
-	}
-
-	if v := o.Deleted; v != 0 {
-		l += 2
-		x := uint64(v)
-		if v < 0 {
-			x = ^x + 1
-		}
-		for n := 0; x >= 0x80 && n < 8; n++ {
-			x >>= 7
-			l++
-		}
-	}
-
-	if o.Secret {
-		l++
-	}
-
-	if o.Cronjewel {
-		l++
-	}
-
-	if l > ColferSizeMax {
-		return l, ColferMax(fmt.Sprintf("colfer: struct model.course exceeds %d bytes", ColferSizeMax))
-	}
-	return l, nil
-}
-
-// MarshalBinary encodes o as Colfer conform encoding.BinaryMarshaler.
-// The error return option is model.ColferMax.
-func (o *Course) MarshalBinary() (data []byte, err error) {
-	l, err := o.MarshalLen()
-	if err != nil {
-		return nil, err
-	}
-	data = make([]byte, l)
-	o.MarshalTo(data)
-	return data, nil
-}
-
-// Unmarshal decodes data as Colfer and returns the number of bytes read.
-// The error return options are io.EOF, model.ColferError and model.ColferMax.
-func (o *Course) Unmarshal(data []byte) (int, error) {
-	if len(data) == 0 {
-		return 0, io.EOF
-	}
-	header := data[0]
-	i := 1
-
-	if header == 0 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Created = int64(x)
-
-		header = data[i]
-		i++
-	} else if header == 0|0x80 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Created = int64(^x + 1)
-
-		header = data[i]
-		i++
-	}
-
-	if header == 1 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Modified = int64(x)
-
-		header = data[i]
-		i++
-	} else if header == 1|0x80 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Modified = int64(^x + 1)
-
-		header = data[i]
-		i++
-	}
-
-	if header == 2 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Expires = int64(x)
-
-		header = data[i]
-		i++
-	} else if header == 2|0x80 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Expires = int64(^x + 1)
-
-		header = data[i]
-		i++
-	}
-
-	if header == 3 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Deleted = int64(x)
-
-		header = data[i]
-		i++
-	} else if header == 3|0x80 {
-		if i+1 >= len(data) {
-			i++
-			goto eof
-		}
-		x := uint64(data[i])
-		i++
-
-		if x >= 0x80 {
-			x &= 0x7f
-			for shift := uint(7); ; shift += 7 {
-				b := uint64(data[i])
-				i++
-				if i >= len(data) {
-					goto eof
-				}
-
-				if b < 0x80 || shift == 56 {
-					x |= b << shift
-					break
-				}
-				x |= (b & 0x7f) << shift
-			}
-		}
-		o.Deleted = int64(^x + 1)
-
-		header = data[i]
-		i++
-	}
-
-	if header == 4 {
-		if i >= len(data) {
-			goto eof
-		}
-		o.Secret = true
-		header = data[i]
-		i++
-	}
-
-	if header == 5 {
-		if i >= len(data) {
-			goto eof
-		}
-		o.Cronjewel = true
-		header = data[i]
-		i++
-	}
-
-	if header != 0x7f {
-		return 0, ColferError(i - 1)
-	}
-	if i < ColferSizeMax {
-		return i, nil
-	}
-eof:
-	if i >= ColferSizeMax {
-		return 0, ColferMax(fmt.Sprintf("colfer: struct model.course size exceeds %d bytes", ColferSizeMax))
-	}
-	return 0, io.EOF
-}
-
-// UnmarshalBinary decodes data as Colfer conform encoding.BinaryUnmarshaler.
-// The error return options are io.EOF, model.ColferError, model.ColferTail and model.ColferMax.
-func (o *Course) UnmarshalBinary(data []byte) error {
-	i, err := o.Unmarshal(data)
-	if i < len(data) && err == nil {
-		return ColferTail(i)
-	}
-	return err
-}
diff --git a/database/namespaces.go b/database/namespaces.go
deleted file mode 100644
index 4dca334..0000000
--- a/database/namespaces.go
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import datastore "github.com/ipfs/go-datastore"
-
-var (
-	// Persistent data that is fetched or gathered, entries may be deleted
-	Cache                 = datastore.NewKey("/Cache")
-	DNSCache              = Cache.ChildString("Dns")
-	IntelCache            = Cache.ChildString("Intel")
-	FileInfoCache         = Cache.ChildString("FileInfo")
-	ProfileCache          = Cache.ChildString("Profile")
-	IPInfoCache           = Cache.ChildString("IPInfo")
-	CertCache             = Cache.ChildString("Cert")
-	CARevocationInfoCache = Cache.ChildString("CARevocationInfo")
-
-	// Volatile, in-memory (recommended) namespace for storing runtime information, cleans itself
-	Run                = datastore.NewKey("/Run")
-	Processes          = Run.ChildString("Processes")
-	OrphanedConnection = Run.ChildString("OrphanedConnections")
-	OrphanedLink       = Run.ChildString("OrphanedLinks")
-	Api                = Run.ChildString("Api")
-	ApiSessions        = Api.ChildString("ApiSessions")
-
-	// Namespace for current device, will be mounted into /Devices/[device]
-	Me = datastore.NewKey("/Me")
-
-	// Holds data of all Devices
-	Devices = datastore.NewKey("/Devices")
-
-	// Holds persistent data
-	Data     = datastore.NewKey("/Data")
-	Profiles = Data.ChildString("Profiles")
-
-	// Holds data distributed by the System (coming from the Community and Devs)
-	Dist         = datastore.NewKey("/Dist")
-	DistProfiles = Dist.ChildString("Profiles")
-	DistUpdates  = Dist.ChildString("Updates")
-
-	// Holds data issued by company
-	Company         = datastore.NewKey("/Company")
-	CompanyProfiles = Company.ChildString("Profiles")
-	CompanyUpdates  = Company.ChildString("Updates")
-
-	// Server
-	// The Authority namespace is used by authoritative servers (Safing or Company) to store data (Intel, Profiles, ...) to be served to clients
-	Authority            = datastore.NewKey("/Authority")
-	AthoritativeIntel    = Authority.ChildString("Intel")
-	AthoritativeProfiles = Authority.ChildString("Profiles")
-	// The Staging namespace is the same as the Authority namespace, but for rolling out new things first to a selected list of clients for testing
-	AuthorityStaging            = datastore.NewKey("/Staging")
-	AthoritativeStagingProfiles = AuthorityStaging.ChildString("Profiles")
-
-	// Holds data of Apps
-	Apps = datastore.NewKey("/Apps")
-
-	// Test & Invalid namespace
-	Tests   = datastore.NewKey("/Tests")
-	Invalid = datastore.NewKey("/Invalid")
-)
diff --git a/database/model/base.go b/database/record/base.go
similarity index 100%
rename from database/model/base.go
rename to database/record/base.go
diff --git a/database/model/base_test.go b/database/record/base_test.go
similarity index 55%
rename from database/model/base_test.go
rename to database/record/base_test.go
index a1fbe84..6e4205d 100644
--- a/database/model/base_test.go
+++ b/database/record/base_test.go
@@ -2,11 +2,11 @@ package model
 
 import "testing"
 
-func TestBaseModel(t *testing.T) {
+func TestBaseRecord(t *testing.T) {
 
 	// check model interface compliance
-	var m Model
-	b := &TestModel{}
+	var m Record
+	b := &TestRecord{}
 	m = b
 	_ = m
 
diff --git a/database/model/formats.go b/database/record/formats.go
similarity index 100%
rename from database/model/formats.go
rename to database/record/formats.go
diff --git a/database/model/key.go b/database/record/key.go
similarity index 100%
rename from database/model/key.go
rename to database/record/key.go
diff --git a/database/model/meta-bench_test.go b/database/record/meta-bench_test.go
similarity index 100%
rename from database/model/meta-bench_test.go
rename to database/record/meta-bench_test.go
diff --git a/database/model/meta-gencode.go b/database/record/meta-gencode.go
similarity index 100%
rename from database/model/meta-gencode.go
rename to database/record/meta-gencode.go
diff --git a/database/model/meta-gencode_test.go b/database/record/meta-gencode_test.go
similarity index 100%
rename from database/model/meta-gencode_test.go
rename to database/record/meta-gencode_test.go
diff --git a/database/model/meta.colf b/database/record/meta.colf
similarity index 100%
rename from database/model/meta.colf
rename to database/record/meta.colf
diff --git a/database/model/meta.gencode b/database/record/meta.gencode
similarity index 100%
rename from database/model/meta.gencode
rename to database/record/meta.gencode
diff --git a/database/model/meta.go b/database/record/meta.go
similarity index 100%
rename from database/model/meta.go
rename to database/record/meta.go
diff --git a/database/model/model.go b/database/record/record.go
similarity index 76%
rename from database/model/model.go
rename to database/record/record.go
index 71fe032..e940dbe 100644
--- a/database/model/model.go
+++ b/database/record/record.go
@@ -1,7 +1,7 @@
 package model
 
-// Model provides an interface for uniformally handling database records.
-type Model interface {
+// Record provides an interface for uniformally handling database records.
+type Record interface {
 	Key() string          // test:config
 	DatabaseName() string // test
 	DatabaseKey() string  // config
diff --git a/database/model/model_test.go b/database/record/record_test.go
similarity index 51%
rename from database/model/model_test.go
rename to database/record/record_test.go
index de732f7..00a9c63 100644
--- a/database/model/model_test.go
+++ b/database/record/record_test.go
@@ -2,15 +2,15 @@ package model
 
 import "sync"
 
-type TestModel struct {
+type TestRecord struct {
 	Base
 	lock sync.Mutex
 }
 
-func (tm *TestModel) Lock() {
+func (tm *TestRecord) Lock() {
 	tm.lock.Lock()
 }
 
-func (tm *TestModel) Unlock() {
+func (tm *TestRecord) Unlock() {
 	tm.lock.Unlock()
 }
diff --git a/database/model/wrapper.go b/database/record/wrapper.go
similarity index 100%
rename from database/model/wrapper.go
rename to database/record/wrapper.go
diff --git a/database/model/wrapper_test.go b/database/record/wrapper_test.go
similarity index 98%
rename from database/model/wrapper_test.go
rename to database/record/wrapper_test.go
index f1c7c4a..45ea13e 100644
--- a/database/model/wrapper_test.go
+++ b/database/record/wrapper_test.go
@@ -10,7 +10,7 @@ import (
 func TestWrapper(t *testing.T) {
 
 	// check model interface compliance
-	var m Model
+	var m Record
 	w := &Wrapper{}
 	m = w
 	_ = m
diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
index 83aa821..21f75b9 100644
--- a/database/storage/badger/badger.go
+++ b/database/storage/badger/badger.go
@@ -7,7 +7,7 @@ import (
 	"github.com/dgraph-io/badger"
 
 	"github.com/Safing/portbase/database/iterator"
-	"github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/query"
 	"github.com/Safing/portbase/database/storage"
 )
@@ -58,7 +58,7 @@ func (b *Badger) Exists(key string) (bool, error) {
 }
 
 // Get returns a database record.
-func (b *Badger) Get(key string) (model.Model, error) {
+func (b *Badger) Get(key string) (record.Record, error) {
 	var item *badger.Item
 
 	err := b.db.View(func(txn *badger.Txn) error {
@@ -93,7 +93,7 @@ func (b *Badger) Get(key string) (model.Model, error) {
 }
 
 // Put stores a record in the database.
-func (b *Badger) Put(m model.Model) error {
+func (b *Badger) Put(m record.Record) error {
 	data, err := m.MarshalRecord()
 	if err != nil {
 		return err
diff --git a/database/storage/interface.go b/database/storage/interface.go
index f6753f9..ae19a90 100644
--- a/database/storage/interface.go
+++ b/database/storage/interface.go
@@ -2,15 +2,15 @@ package storage
 
 import (
 	"github.com/Safing/portbase/database/iterator"
-	"github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/query"
 )
 
 // Interface defines the database storage API.
 type Interface interface {
 	Exists(key string) (bool, error)
-	Get(key string) (model.Model, error)
-	Put(m model.Model) error
+	Get(key string) (record.Record, error)
+	Put(m record.Record) error
 	Delete(key string) error
 	Query(q *query.Query) (*iterator.Iterator, error)
 
diff --git a/database/storage/sinkhole/sinkhole.go b/database/storage/sinkhole/sinkhole.go
index fe6ec4f..5a5189e 100644
--- a/database/storage/sinkhole/sinkhole.go
+++ b/database/storage/sinkhole/sinkhole.go
@@ -4,7 +4,7 @@ import (
 	"errors"
 
 	"github.com/Safing/portbase/database/iterator"
-	"github.com/Safing/portbase/database/model"
+	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/query"
 	"github.com/Safing/portbase/database/storage"
 )
@@ -31,12 +31,12 @@ func (s *Sinkhole) Exists(key string) (bool, error) {
 }
 
 // Get returns a database record.
-func (s *Sinkhole) Get(key string) (model.Model, error) {
+func (s *Sinkhole) Get(key string) (record.Record, error) {
 	return nil, storage.ErrNotFound
 }
 
 // Put stores a record in the database.
-func (s *Sinkhole) Put(m model.Model) error {
+func (s *Sinkhole) Put(m record.Record) error {
 	return nil
 }
 

From 7ad09b60c1618076aff1175e4f3f8bbb58d0d9fd Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 6 Sep 2018 08:52:24 +0200
Subject: [PATCH 11/20] Finish renaming model package to record

---
 database/record/base.go              | 2 +-
 database/record/base_test.go         | 2 +-
 database/record/formats.go           | 2 +-
 database/record/key.go               | 2 +-
 database/record/meta-bench_test.go   | 2 +-
 database/record/meta-gencode.go      | 2 +-
 database/record/meta-gencode_test.go | 2 +-
 database/record/meta.colf            | 2 +-
 database/record/meta.go              | 2 +-
 database/record/record.go            | 2 +-
 database/record/record_test.go       | 2 +-
 database/record/wrapper.go           | 2 +-
 database/record/wrapper_test.go      | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/database/record/base.go b/database/record/base.go
index 03ed22a..30fcfc1 100644
--- a/database/record/base.go
+++ b/database/record/base.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import (
 	"errors"
diff --git a/database/record/base_test.go b/database/record/base_test.go
index 6e4205d..f207bb1 100644
--- a/database/record/base_test.go
+++ b/database/record/base_test.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import "testing"
 
diff --git a/database/record/formats.go b/database/record/formats.go
index 6718593..d453337 100644
--- a/database/record/formats.go
+++ b/database/record/formats.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import (
 	"github.com/Safing/portbase/formats/dsd"
diff --git a/database/record/key.go b/database/record/key.go
index 81dee11..b02eecf 100644
--- a/database/record/key.go
+++ b/database/record/key.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import (
 	"strings"
diff --git a/database/record/meta-bench_test.go b/database/record/meta-bench_test.go
index 262aaec..2c62083 100644
--- a/database/record/meta-bench_test.go
+++ b/database/record/meta-bench_test.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 // Benchmark:
 // BenchmarkAllocateBytes-8                	2000000000	         0.76 ns/op
diff --git a/database/record/meta-gencode.go b/database/record/meta-gencode.go
index 7c3f494..d79edff 100644
--- a/database/record/meta-gencode.go
+++ b/database/record/meta-gencode.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import (
 	"fmt"
diff --git a/database/record/meta-gencode_test.go b/database/record/meta-gencode_test.go
index 4f3bdcb..baea558 100644
--- a/database/record/meta-gencode_test.go
+++ b/database/record/meta-gencode_test.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import (
 	"reflect"
diff --git a/database/record/meta.colf b/database/record/meta.colf
index 7cc99f4..0072e92 100644
--- a/database/record/meta.colf
+++ b/database/record/meta.colf
@@ -1,4 +1,4 @@
-package model
+package record
 
 type course struct {
   Created   int64
diff --git a/database/record/meta.go b/database/record/meta.go
index 057a8a4..c561a17 100644
--- a/database/record/meta.go
+++ b/database/record/meta.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import "time"
 
diff --git a/database/record/record.go b/database/record/record.go
index e940dbe..c002f25 100644
--- a/database/record/record.go
+++ b/database/record/record.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 // Record provides an interface for uniformally handling database records.
 type Record interface {
diff --git a/database/record/record_test.go b/database/record/record_test.go
index 00a9c63..f5e315d 100644
--- a/database/record/record_test.go
+++ b/database/record/record_test.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import "sync"
 
diff --git a/database/record/wrapper.go b/database/record/wrapper.go
index bdb9c2a..3f73263 100644
--- a/database/record/wrapper.go
+++ b/database/record/wrapper.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import (
 	"errors"
diff --git a/database/record/wrapper_test.go b/database/record/wrapper_test.go
index 45ea13e..476ff2f 100644
--- a/database/record/wrapper_test.go
+++ b/database/record/wrapper_test.go
@@ -1,4 +1,4 @@
-package model
+package record
 
 import (
 	"bytes"

From b8e7f90dbea0ad2ecbcd93e8db498a51fca1d1c2 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 6 Sep 2018 19:06:13 +0200
Subject: [PATCH 12/20] Continue work on database module

---
 database/controller.go               |  95 +++--
 database/database.go                 | 156 ++++----
 database/databases.go                | 123 +++---
 database/dbmodule/db.go              |  29 ++
 database/easyquery.go                |  84 -----
 database/easyquery_test.go           |  68 ----
 database/interface.go                |  64 +++-
 database/location.go                 |  14 +
 database/model.go                    |  64 ----
 database/model_test.go               | 108 ------
 database/queries.go                  |  37 --
 database/record/base.go              |   5 +
 database/record/meta-bench_test.go   |  56 +--
 database/record/meta-gencode.go      |  72 ++--
 database/record/meta-gencode_test.go |   8 +-
 database/record/meta.go              |  56 +--
 database/record/record.go            |   2 +
 database/record/wrapper.go           |  22 ++
 database/registry.go                 | 145 ++++++++
 database/storage/interface.go        |   3 +-
 database/storage/storages.go         |   5 +
 database/subscriptions.go            | 536 +++++++++++++--------------
 database/subscriptions_test.go       | 198 +++++-----
 database/wrapper.go                  |  17 -
 database/wrapper_test.go             |  68 ----
 25 files changed, 962 insertions(+), 1073 deletions(-)
 create mode 100644 database/dbmodule/db.go
 delete mode 100644 database/easyquery.go
 delete mode 100644 database/easyquery_test.go
 create mode 100644 database/location.go
 delete mode 100644 database/model.go
 delete mode 100644 database/model_test.go
 delete mode 100644 database/queries.go
 create mode 100644 database/registry.go
 delete mode 100644 database/wrapper.go
 delete mode 100644 database/wrapper_test.go

diff --git a/database/controller.go b/database/controller.go
index 499b746..428f319 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -1,42 +1,89 @@
 package database
 
 import (
-  "github.com/Safing/portbase/database/record"
+	"sync"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/Safing/portbase/database/iterator"
+	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/record"
+	"github.com/Safing/portbase/database/storage"
 )
 
+// A Controller takes care of all the extra database logic.
 type Controller struct {
-  storage
-  writeLock sync.RWMutex
-  readLock sync.RWMutex
-  migrating *abool.AtomicBool
+	storage   storage.Interface
+	writeLock sync.RWMutex
+	readLock  sync.RWMutex
+	migrating *abool.AtomicBool
 }
 
-func NewController() (*Controller, error) {
-
+// newController creates a new controller for a storage.
+func newController(storageInt storage.Interface) (*Controller, error) {
+	return &Controller{
+		storage:   storageInt,
+		migrating: abool.NewBool(false),
+	}, nil
 }
 
-	// Retrieve
-func (c *Controller) Exists(key string) (bool, error) {}
-func (c *Controller) Get(key string) (record.Record, error) {}
+// Get return the record with the given key.
+func (c *Controller) Get(key string) (record.Record, error) {
+	r, err := c.storage.Get(key)
+	if err != nil {
+		return nil, err
+	}
 
-// Modify
-func (c *Controller) Create(model record.Record) error {}
-// create when not exists
-func (c *Controller) Update(model record.Record) error {}
-// update, create if not exists.
-func (c *Controller) UpdateOrCreate(model record.Record) error {}
-func (c *Controller) Delete(key string) error {}
+	if !r.Meta().CheckValidity(time.Now().Unix()) {
+		return nil, ErrNotFound
+	}
+
+	return r, nil
+}
+
+// Put saves a record in the database.
+func (c *Controller) Put(r record.Record) error {
+	return c.storage.Put(r)
+}
+
+func (c *Controller) Delete(key string) error {
+	r, err := c.Get(key)
+	if err != nil {
+		return err
+	}
+	r.Meta().Deleted = time.Now().Unix()
+	return c.Put(r)
+}
 
 // Partial
 // What happens if I mutate a value that does not yet exist? How would I know its type?
-func (c *Controller) InsertPartial(key string, partialObject interface{}) {}
-func (c *Controller) InsertValue(key string, attribute string, value interface{}) {}
+func (c *Controller) InsertPartial(key string, partialObject interface{}) error {
+	return nil
+}
+
+func (c *Controller) InsertValue(key string, attribute string, value interface{}) error {
+	return nil
+}
 
 // Query
-func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {}
+func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {
+	return nil, nil
+}
 
 // Meta
-func (c *Controller) SetAbsoluteExpiry(key string, time int64) {}
-func (c *Controller) SetRelativateExpiry(key string, duration int64) {}
-func (c *Controller) MakeCrownJewel(key string) {}
-func (c *Controller) MakeSecret(key string) {}
+func (c *Controller) SetAbsoluteExpiry(key string, time int64) error {
+	return nil
+}
+
+func (c *Controller) SetRelativateExpiry(key string, duration int64) error {
+	return nil
+}
+
+func (c *Controller) MakeCrownJewel(key string) error {
+	return nil
+}
+
+func (c *Controller) MakeSecret(key string) error {
+	return nil
+}
diff --git a/database/database.go b/database/database.go
index 863604e..78373f4 100644
--- a/database/database.go
+++ b/database/database.go
@@ -4,17 +4,13 @@ package database
 
 import (
 	"errors"
-	"fmt"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/Safing/safing-core/database/dbutils"
-	"github.com/Safing/safing-core/log"
-	"github.com/Safing/safing-core/meta"
 )
 
-var ErrNotFound = errors.New("database: entry could not be found")
+// Errors
+var (
+	ErrNotFound         = errors.New("database: entry could not be found")
+	ErrPermissionDenied = errors.New("database: access to record denied")
+)
 
 func init() {
 	// if strings.HasSuffix(os.Args[0], ".test") {
@@ -59,84 +55,66 @@ func init() {
 //   return db.Close()
 // }
 
-func Get(key *ds.Key) (Model, error) {
-	data, err := db.Get(*key)
-	if err != nil {
-		switch err {
-		case ds.ErrNotFound:
-			return nil, ErrNotFound
-		default:
-			return nil, err
-		}
-	}
-	model, ok := data.(Model)
-	if !ok {
-		return nil, errors.New("database did not return model")
-	}
-	return model, nil
-}
+// func Get(key *ds.Key) (Model, error) {
+// 	data, err := db.Get(*key)
+// 	if err != nil {
+// 		switch err {
+// 		case ds.ErrNotFound:
+// 			return nil, ErrNotFound
+// 		default:
+// 			return nil, err
+// 		}
+// 	}
+// 	model, ok := data.(Model)
+// 	if !ok {
+// 		return nil, errors.New("database did not return model")
+// 	}
+// 	return model, nil
+// }
 
-func GetAndEnsureModel(namespace *ds.Key, name string, model Model) (Model, error) {
-	newKey := namespace.ChildString(getTypeName(model)).Instance(name)
-
-	data, err := Get(&newKey)
-	if err != nil {
-		return nil, err
-	}
-
-	newModel, err := EnsureModel(data, model)
-	if err != nil {
-		return nil, err
-	}
-
-	newModel.SetKey(&newKey)
-
-	return newModel, nil
-}
-
-func Has(key ds.Key) (exists bool, err error) {
-	return db.Has(key)
-}
-
-func Create(key ds.Key, model Model) (err error) {
-	handleCreateSubscriptions(model)
-	err = db.Put(key, model)
-	if err != nil {
-		log.Tracef("database: failed to create entry %s: %s", key, err)
-	}
-	return err
-}
-
-func Update(key ds.Key, model Model) (err error) {
-	handleUpdateSubscriptions(model)
-	err = db.Put(key, model)
-	if err != nil {
-		log.Tracef("database: failed to update entry %s: %s", key, err)
-	}
-	return err
-}
-
-func Delete(key ds.Key) (err error) {
-	handleDeleteSubscriptions(&key)
-	return db.Delete(key)
-}
-
-func Query(q dsq.Query) (dsq.Results, error) {
-	return db.Query(q)
-}
-
-func RawGet(key ds.Key) (*dbutils.Wrapper, error) {
-	data, err := db.Get(key)
-	if err != nil {
-		return nil, err
-	}
-	wrapped, ok := data.(*dbutils.Wrapper)
-	if !ok {
-		return nil, errors.New("returned data is not a wrapper")
-	}
-	return wrapped, nil
-}
-
-func RawPut(key ds.Key, value interface{}) error {
-	return db.Put(key, value)
-}
+// func Has(key ds.Key) (exists bool, err error) {
+// 	return db.Has(key)
+// }
+//
+// func Create(key ds.Key, model Model) (err error) {
+// 	handleCreateSubscriptions(model)
+// 	err = db.Put(key, model)
+// 	if err != nil {
+// 		log.Tracef("database: failed to create entry %s: %s", key, err)
+// 	}
+// 	return err
+// }
+//
+// func Update(key ds.Key, model Model) (err error) {
+// 	handleUpdateSubscriptions(model)
+// 	err = db.Put(key, model)
+// 	if err != nil {
+// 		log.Tracef("database: failed to update entry %s: %s", key, err)
+// 	}
+// 	return err
+// }
+//
+// func Delete(key ds.Key) (err error) {
+// 	handleDeleteSubscriptions(&key)
+// 	return db.Delete(key)
+// }
+//
+// func Query(q dsq.Query) (dsq.Results, error) {
+// 	return db.Query(q)
+// }
+//
+// func RawGet(key ds.Key) (*dbutils.Wrapper, error) {
+// 	data, err := db.Get(key)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	wrapped, ok := data.(*dbutils.Wrapper)
+// 	if !ok {
+// 		return nil, errors.New("returned data is not a wrapper")
+// 	}
+// 	return wrapped, nil
+// }
+//
+// func RawPut(key ds.Key, value interface{}) error {
+// 	return db.Put(key, value)
+// }
diff --git a/database/databases.go b/database/databases.go
index b9cc0e9..6170c71 100644
--- a/database/databases.go
+++ b/database/databases.go
@@ -1,56 +1,95 @@
 package database
 
+import (
+	"errors"
+	"sync"
+  "fmt"
+  "path"
 
-var (
-  databases = make(map[string]*Controller)
-  databasesLock sync.Mutex
+  "github.com/Safing/portbase/database/storage"
+	"github.com/Safing/portbase/database/record"
 )
 
-func getDatabase(name string) *Controller {
-  databasesLock.Lock()
-  defer   databasesLock.Unlock()
-  storage, ok := databases[name]
-  if ok {
-    return
-  }
-}
+var (
+	databases     = make(map[string]*Controller)
+	databasesLock sync.Mutex
+)
 
-func databaseExists(name string) (exists bool) {
-  // check if folder exists
-  return true
-}
-
-// CreateDatabase creates a new database with given name and type.
-func CreateDatabase(name string, storageType string) error {
-  databasesLock.Lock()
-  defer   databasesLock.Unlock()
-  _, ok := databases[name]
-  if ok {
-    return errors.New("database with this name already loaded.")
-  }
-  if databaseExists(name) {
-    return errors.New("database with this name already exists.")
-  }
-
-  iface, err := startDatabase(name)
+func splitKeyAndGetDatabase(key string) (dbKey string, db *Controller, err error) {
+  var dbName string
+  dbName, dbKey = record.ParseKey(key)
+  db, err = getDatabase(dbName)
   if err != nil {
-    return err
+    return "", nil, err
   }
-  databases[name] = iface
-  return nil
+  return
+}
+
+func getDatabase(name string) (*Controller, error) {
+  if !initialized.IsSet() {
+    return nil, errors.New("database not initialized")
+  }
+
+	databasesLock.Lock()
+	defer databasesLock.Unlock()
+
+  // return database if already started
+	db, ok := databases[name]
+	if ok {
+    return db, nil
+	}
+
+  registryLock.Lock()
+  defer registryLock.Unlock()
+
+  // check if database exists at all
+  registeredDB, ok := registry[name]
+  if !ok {
+    return nil, fmt.Errorf(`database "%s" not registered`, name)
+  }
+
+  // start database
+  storageInt, err := storage.StartDatabase(name, registeredDB.StorageType, path.Join(rootDir, name, registeredDB.StorageType))
+  if err != nil {
+    return nil, fmt.Errorf(`could not start database %s (type %s): %s`, name, registeredDB.StorageType, err)
+  }
+
+  db, err = newController(storageInt)
+  if err != nil {
+    return nil, fmt.Errorf(`could not create controller for database %s: %s`, name, err)
+  }
+
+  databases[name] = db
+  return db, nil
 }
 
 // InjectDatabase injects an already running database into the system.
-func InjectDatabase(name string, iface *storage.Interface) error {
-  databasesLock.Lock()
-  defer   databasesLock.Unlock()
-  _, ok := databases[name]
-  if ok {
-    return errors.New("database with this name already loaded.")
+func InjectDatabase(name string, storageInt storage.Interface) error {
+	databasesLock.Lock()
+	defer databasesLock.Unlock()
+
+	_, ok := databases[name]
+	if ok {
+		return errors.New(`database "%s" already loaded`)
+	}
+
+  registryLock.Lock()
+  defer registryLock.Unlock()
+
+  // check if database is registered
+  registeredDB, ok := registry[name]
+  if !ok {
+    return fmt.Errorf(`database "%s" not registered`, name)
   }
-  if databaseExists(name) {
-    return errors.New("database with this name already exists.")
+  if registeredDB.StorageType != "injected" {
+    return fmt.Errorf(`database not of type "injected"`)
   }
-  databases[name] = iface
-  return nil
+
+  db, err := newController(storageInt)
+  if err != nil {
+    return fmt.Errorf(`could not create controller for database %s: %s`, name, err)
+  }
+
+	databases[name] = db
+	return nil
 }
diff --git a/database/dbmodule/db.go b/database/dbmodule/db.go
new file mode 100644
index 0000000..6c38c1f
--- /dev/null
+++ b/database/dbmodule/db.go
@@ -0,0 +1,29 @@
+package dbmodule
+
+import (
+  "github.com/Safing/portbase/database"
+)
+
+var (
+	databaseDir string
+)
+
+func init() {
+  flag.StringVar(&databaseDir, "db", "", "set database directory")
+
+  modules.Register("database", prep, start, stop)
+}
+
+func prep() error {
+  if databaseDir == "" {
+    return errors.New("no database location specified, set with `-db=/path/to/db`")
+  }
+}
+
+func start() error {
+  return database.Initialize(databaseDir)
+}
+
+func stop() {
+  return database.Shutdown()
+}
diff --git a/database/easyquery.go b/database/easyquery.go
deleted file mode 100644
index ee4db82..0000000
--- a/database/easyquery.go
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-
-	dsq "github.com/ipfs/go-datastore/query"
-)
-
-type FilterMaxDepth struct {
-	MaxDepth int
-}
-
-func (f FilterMaxDepth) Filter(entry dsq.Entry) bool {
-	return strings.Count(entry.Key, "/") <= f.MaxDepth
-}
-
-type FilterKeyLength struct {
-	Length int
-}
-
-func (f FilterKeyLength) Filter(entry dsq.Entry) bool {
-	return len(entry.Key) == f.Length
-}
-
-func EasyQueryIterator(subscriptionKey string) (dsq.Results, error) {
-	query := dsq.Query{}
-
-	namespaces := strings.Split(subscriptionKey, "/")[1:]
-	lastSpace := ""
-	if len(namespaces) != 0 {
-		lastSpace = namespaces[len(namespaces)-1]
-	}
-
-	switch {
-	case lastSpace == "":
-		// get all children
-		query.Prefix = subscriptionKey
-	case strings.HasPrefix(lastSpace, "*"):
-		// get children to defined depth
-		query.Prefix = strings.Trim(subscriptionKey, "*")
-		query.Filters = []dsq.Filter{
-			FilterMaxDepth{len(lastSpace) + len(namespaces) - 1},
-		}
-	case strings.Contains(lastSpace, ":"):
-		query.Prefix = subscriptionKey
-		query.Filters = []dsq.Filter{
-			FilterKeyLength{len(query.Prefix)},
-		}
-	default:
-		// get only from this location and this type
-		query.Prefix = subscriptionKey + ":"
-		query.Filters = []dsq.Filter{
-			FilterMaxDepth{len(namespaces)},
-		}
-	}
-
-	// log.Tracef("easyquery: %s has prefix %s", subscriptionKey, query.Prefix)
-
-	results, err := db.Query(query)
-	if err != nil {
-		return nil, errors.New(fmt.Sprintf("easyquery: %s", err))
-	}
-
-	return results, nil
-}
-
-func EasyQuery(subscriptionKey string) (*[]dsq.Entry, error) {
-
-	results, err := EasyQueryIterator(subscriptionKey)
-	if err != nil {
-		return nil, err
-	}
-
-	entries, err := results.Rest()
-	if err != nil {
-		return nil, errors.New(fmt.Sprintf("easyquery: %s", err))
-	}
-
-	return &entries, nil
-}
diff --git a/database/easyquery_test.go b/database/easyquery_test.go
deleted file mode 100644
index 914c0b8..0000000
--- a/database/easyquery_test.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"testing"
-
-	datastore "github.com/ipfs/go-datastore"
-)
-
-func testQuery(t *testing.T, queryString string, expecting []string) {
-
-	entries, err := EasyQuery(queryString)
-	if err != nil {
-		t.Errorf("error in query %s: %s", queryString, err)
-	}
-
-	totalExcepted := len(expecting)
-	total := 0
-	fail := false
-
-	keys := datastore.EntryKeys(*entries)
-
-resultLoop:
-	for _, key := range keys {
-		total++
-		for _, expectedName := range expecting {
-			if key.Name() == expectedName {
-				continue resultLoop
-			}
-		}
-		fail = true
-		break
-	}
-
-	if !fail && total == totalExcepted {
-		return
-	}
-
-	t.Errorf("Query %s got %s, expected %s", queryString, keys, expecting)
-
-}
-
-func TestEasyQuery(t *testing.T) {
-
-	// setup test data
-	(&(TestingModel{})).CreateInNamespace("EasyQuery", "1")
-	(&(TestingModel{})).CreateInNamespace("EasyQuery", "2")
-	(&(TestingModel{})).CreateInNamespace("EasyQuery", "3")
-	(&(TestingModel{})).CreateInNamespace("EasyQuery/A", "4")
-	(&(TestingModel{})).CreateInNamespace("EasyQuery/A/B", "5")
-	(&(TestingModel{})).CreateInNamespace("EasyQuery/A/B/C", "6")
-	(&(TestingModel{})).CreateInNamespace("EasyQuery/A/B/C/D", "7")
-
-	(&(TestingModel{})).CreateWithTypeName("EasyQuery", "ConfigModel", "X")
-	(&(TestingModel{})).CreateWithTypeName("EasyQuery", "ConfigModel", "Y")
-	(&(TestingModel{})).CreateWithTypeName("EasyQuery/A", "ConfigModel", "Z")
-
-	testQuery(t, "/Tests/EasyQuery/TestingModel", []string{"1", "2", "3"})
-	testQuery(t, "/Tests/EasyQuery/TestingModel:1", []string{"1"})
-
-	testQuery(t, "/Tests/EasyQuery/ConfigModel", []string{"X", "Y"})
-	testQuery(t, "/Tests/EasyQuery/ConfigModel:Y", []string{"Y"})
-
-	testQuery(t, "/Tests/EasyQuery/A/", []string{"Z", "4", "5", "6", "7"})
-	testQuery(t, "/Tests/EasyQuery/A/B/**", []string{"5", "6"})
-
-}
diff --git a/database/interface.go b/database/interface.go
index dd56955..c765559 100644
--- a/database/interface.go
+++ b/database/interface.go
@@ -1,27 +1,69 @@
 package database
 
+import (
+	"github.com/Safing/portbase/database/record"
+)
+
 // Interface provides a method to access the database with attached options.
-type Interface struct {}
+type Interface struct {
+	options *Options
+}
 
 // Options holds options that may be set for an Interface instance.
 type Options struct {
-  Local bool
-  Internal bool
-  AlwaysMakeSecret bool
-  AlwaysMakeCrownjewel bool
+	Local                bool
+	Internal             bool
+	AlwaysMakeSecret     bool
+	AlwaysMakeCrownjewel bool
 }
 
 // NewInterface returns a new Interface to the database.
 func NewInterface(opts *Options) *Interface {
-  return &Interface{
-    local: local,
-    internal: internal,
-  }
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	return &Interface{
+		options: opts,
+	}
 }
 
+// Exists return whether a record with the given key exists.
+func (i *Interface) Exists(key string) (bool, error) {
+	_, err := i.getRecord(key)
+	if err != nil {
+		if err == ErrNotFound {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// Get return the record with the given key.
 func (i *Interface) Get(key string) (record.Record, error) {
+	r, err := i.getRecord(key)
+	if err != nil {
+		return nil, err
+	}
 
-  controller
+	if !r.Meta().CheckPermission(i.options.Local, i.options.Internal) {
+		return nil, ErrPermissionDenied
+	}
 
-  return nil, nil
+	return r, nil
+}
+
+func (i *Interface) getRecord(key string) (record.Record, error) {
+	dbKey, db, err := splitKeyAndGetDatabase(key)
+	if err != nil {
+		return nil, err
+	}
+
+	r, err := db.Get(dbKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
 }
diff --git a/database/location.go b/database/location.go
new file mode 100644
index 0000000..c0fdb78
--- /dev/null
+++ b/database/location.go
@@ -0,0 +1,14 @@
+package database
+
+import (
+	"path"
+)
+
+var (
+	rootDir string
+)
+
+// getLocation returns the storage location for the given name and type.
+func getLocation(name, storageType string) (location string, err error) {
+	return path.Join(rootDir, name, storageType), nil
+}
diff --git a/database/model.go b/database/model.go
deleted file mode 100644
index 1aa7634..0000000
--- a/database/model.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/Safing/safing-core/database/dbutils"
-	"github.com/Safing/safing-core/formats/dsd"
-)
-
-// Model Registration
-
-var (
-	registeredModels     = make(map[string]func() Model)
-	registeredModelsLock sync.RWMutex
-)
-
-func RegisterModel(model Model, constructor func() Model) {
-	registeredModelsLock.Lock()
-	defer registeredModelsLock.Unlock()
-	registeredModels[fmt.Sprintf("%T", model)] = constructor
-}
-
-func NewModel(model Model) (Model, error) {
-	registeredModelsLock.RLock()
-	defer registeredModelsLock.RUnlock()
-	constructor, ok := registeredModels[fmt.Sprintf("%T", model)]
-	if !ok {
-		return nil, fmt.Errorf("database: cannot create new %T, not registered", model)
-	}
-	return constructor(), nil
-}
-
-func EnsureModel(uncertain, model Model) (Model, error) {
-	wrappedObj, ok := uncertain.(*dbutils.Wrapper)
-	if !ok {
-		return uncertain, nil
-	}
-	newModel, err := NewModel(model)
-	if err != nil {
-		return nil, err
-	}
-	_, err = dsd.Load(wrappedObj.Data, &newModel)
-	if err != nil {
-		return nil, fmt.Errorf("database: failed to unwrap %T: %s", model, err)
-	}
-	newModel.SetKey(wrappedObj.GetKey())
-	model = newModel
-	return newModel, nil
-}
-
-func SilentEnsureModel(uncertain, model Model) Model {
-	obj, err := EnsureModel(uncertain, model)
-	if err != nil {
-		return nil
-	}
-	return obj
-}
-
-func NewMismatchError(got, expected interface{}) error {
-	return fmt.Errorf("database: entry (%T) does not match expected model (%T)", got, expected)
-}
diff --git a/database/model_test.go b/database/model_test.go
deleted file mode 100644
index 9d1f486..0000000
--- a/database/model_test.go
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"testing"
-
-	datastore "github.com/ipfs/go-datastore"
-)
-
-type TestingModel struct {
-	Base
-	Name  string
-	Value string
-}
-
-var testingModel *TestingModel
-
-func init() {
-	RegisterModel(testingModel, func() Model { return new(TestingModel) })
-}
-
-func (m *TestingModel) Create(name string) error {
-	return m.CreateObject(&Tests, name, m)
-}
-
-func (m *TestingModel) CreateInNamespace(namespace string, name string) error {
-	testsNamescace := Tests.ChildString(namespace)
-	return m.CreateObject(&testsNamescace, name, m)
-}
-
-func (m *TestingModel) CreateWithTypeName(namespace string, typeName string, name string) error {
-	customNamespace := Tests.ChildString(namespace).ChildString(typeName).Instance(name)
-
-	m.dbKey = &customNamespace
-	handleCreateSubscriptions(m)
-	return Create(*m.dbKey, m)
-}
-
-func (m *TestingModel) Save() error {
-	return m.SaveObject(m)
-}
-
-func GetTestingModel(name string) (*TestingModel, error) {
-	return GetTestingModelFromNamespace(&Tests, name)
-}
-
-func GetTestingModelFromNamespace(namespace *datastore.Key, name string) (*TestingModel, error) {
-	object, err := GetAndEnsureModel(namespace, name, testingModel)
-	if err != nil {
-		return nil, err
-	}
-	model, ok := object.(*TestingModel)
-	if !ok {
-		return nil, NewMismatchError(object, testingModel)
-	}
-	return model, nil
-}
-
-func TestModel(t *testing.T) {
-
-	// create
-	m := TestingModel{
-		Name:  "a",
-		Value: "b",
-	}
-	// newKey := datastore.NewKey("/Tests/TestingModel:test")
-	// m.dbKey = &newKey
-	// err := Put(*m.dbKey, m)
-	err := m.Create("")
-	if err != nil {
-		t.Errorf("database test: could not create object: %s", err)
-	}
-
-	// get
-	o, err := GetTestingModel(m.dbKey.Name())
-	if err != nil {
-		t.Errorf("database test: failed to get model: %s (%s)", err, m.dbKey.Name())
-	}
-
-	// check fetched object
-	if o.Name != "a" || o.Value != "b" {
-		t.Errorf("database test: values do not match: got Name=%s and Value=%s", o.Name, o.Value)
-	}
-
-	// o, err := Get(*m.dbKey)
-	// if err != nil {
-	//   t.Errorf("database: could not get object: %s", err)
-	// }
-	// n, ok := o.(*TestingModel)
-	// if !ok {
-	//   t.Errorf("database: wrong type, got type %T from %s", o, m.dbKey.String())
-	// }
-
-	// save
-	o.Value = "c"
-	err = o.Save()
-	if err != nil {
-		t.Errorf("database test: could not save object: %s", err)
-	}
-
-	// delete
-	err = o.Delete()
-	if err != nil {
-		t.Errorf("database test: could not delete object: %s", err)
-	}
-
-}
diff --git a/database/queries.go b/database/queries.go
deleted file mode 100644
index 3c5ea13..0000000
--- a/database/queries.go
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"time"
-
-	"github.com/Safing/safing-core/formats/dsd"
-	"github.com/Safing/safing-core/log"
-
-	dsq "github.com/ipfs/go-datastore/query"
-)
-
-func init() {
-	// go dumper()
-}
-
-func dumper() {
-	for {
-		time.Sleep(10 * time.Second)
-		result, err := db.Query(dsq.Query{Prefix: "/Run/Process"})
-		if err != nil {
-			log.Warningf("Query failed: %s", err)
-			continue
-		}
-		log.Infof("Dumping all processes:")
-		for model, ok := result.NextSync(); ok; model, ok = result.NextSync() {
-			bytes, err := dsd.Dump(model, dsd.AUTO)
-			if err != nil {
-				log.Warningf("Error dumping: %s", err)
-				continue
-			}
-			log.Info(string(bytes))
-		}
-		log.Infof("END")
-	}
-}
diff --git a/database/record/base.go b/database/record/base.go
index 30fcfc1..d1e2b04 100644
--- a/database/record/base.go
+++ b/database/record/base.go
@@ -85,3 +85,8 @@ func (b *Base) MarshalRecord() ([]byte, error) {
 
 	return c.CompileData(), nil
 }
+
+// IsWrapped returns whether the record is a Wrapper.
+func (b *Base) IsWrapped() bool {
+	return false
+}
diff --git a/database/record/meta-bench_test.go b/database/record/meta-bench_test.go
index 2c62083..ca845c6 100644
--- a/database/record/meta-bench_test.go
+++ b/database/record/meta-bench_test.go
@@ -32,10 +32,10 @@ import (
 
 var (
 	testMeta = &Meta{
-		created:   time.Now().Unix(),
-		modified:  time.Now().Unix(),
-		expires:   time.Now().Unix(),
-		deleted:   time.Now().Unix(),
+		Created:   time.Now().Unix(),
+		Modified:  time.Now().Unix(),
+		Expires:   time.Now().Unix(),
+		Deleted:   time.Now().Unix(),
 		secret:    true,
 		cronjewel: true,
 	}
@@ -65,10 +65,10 @@ func BenchmarkMetaSerializeContainer(b *testing.B) {
 	// Start benchmark
 	for i := 0; i < b.N; i++ {
 		c := container.New()
-		c.AppendNumber(uint64(testMeta.created))
-		c.AppendNumber(uint64(testMeta.modified))
-		c.AppendNumber(uint64(testMeta.expires))
-		c.AppendNumber(uint64(testMeta.deleted))
+		c.AppendNumber(uint64(testMeta.Created))
+		c.AppendNumber(uint64(testMeta.Modified))
+		c.AppendNumber(uint64(testMeta.Expires))
+		c.AppendNumber(uint64(testMeta.Deleted))
 		switch {
 		case testMeta.secret && testMeta.cronjewel:
 			c.AppendNumber(3)
@@ -87,10 +87,10 @@ func BenchmarkMetaUnserializeContainer(b *testing.B) {
 
 	// Setup
 	c := container.New()
-	c.AppendNumber(uint64(testMeta.created))
-	c.AppendNumber(uint64(testMeta.modified))
-	c.AppendNumber(uint64(testMeta.expires))
-	c.AppendNumber(uint64(testMeta.deleted))
+	c.AppendNumber(uint64(testMeta.Created))
+	c.AppendNumber(uint64(testMeta.Modified))
+	c.AppendNumber(uint64(testMeta.Expires))
+	c.AppendNumber(uint64(testMeta.Deleted))
 	switch {
 	case testMeta.secret && testMeta.cronjewel:
 		c.AppendNumber(3)
@@ -113,25 +113,25 @@ func BenchmarkMetaUnserializeContainer(b *testing.B) {
 		var num uint64
 		c := container.New(encodedData)
 		num, err = c.GetNextN64()
-		newMeta.created = int64(num)
+		newMeta.Created = int64(num)
 		if err != nil {
 			b.Errorf("could not decode: %s", err)
 			return
 		}
 		num, err = c.GetNextN64()
-		newMeta.modified = int64(num)
+		newMeta.Modified = int64(num)
 		if err != nil {
 			b.Errorf("could not decode: %s", err)
 			return
 		}
 		num, err = c.GetNextN64()
-		newMeta.expires = int64(num)
+		newMeta.Expires = int64(num)
 		if err != nil {
 			b.Errorf("could not decode: %s", err)
 			return
 		}
 		num, err = c.GetNextN64()
-		newMeta.deleted = int64(num)
+		newMeta.Deleted = int64(num)
 		if err != nil {
 			b.Errorf("could not decode: %s", err)
 			return
@@ -166,22 +166,22 @@ func BenchmarkMetaSerializeVarInt(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		encoded := make([]byte, 33)
 		offset := 0
-		data := varint.Pack64(uint64(testMeta.created))
+		data := varint.Pack64(uint64(testMeta.Created))
 		for _, part := range data {
 			encoded[offset] = part
 			offset++
 		}
-		data = varint.Pack64(uint64(testMeta.modified))
+		data = varint.Pack64(uint64(testMeta.Modified))
 		for _, part := range data {
 			encoded[offset] = part
 			offset++
 		}
-		data = varint.Pack64(uint64(testMeta.expires))
+		data = varint.Pack64(uint64(testMeta.Expires))
 		for _, part := range data {
 			encoded[offset] = part
 			offset++
 		}
-		data = varint.Pack64(uint64(testMeta.deleted))
+		data = varint.Pack64(uint64(testMeta.Deleted))
 		for _, part := range data {
 			encoded[offset] = part
 			offset++
@@ -207,22 +207,22 @@ func BenchmarkMetaUnserializeVarInt(b *testing.B) {
 	// Setup
 	encoded := make([]byte, 33)
 	offset := 0
-	data := varint.Pack64(uint64(testMeta.created))
+	data := varint.Pack64(uint64(testMeta.Created))
 	for _, part := range data {
 		encoded[offset] = part
 		offset++
 	}
-	data = varint.Pack64(uint64(testMeta.modified))
+	data = varint.Pack64(uint64(testMeta.Modified))
 	for _, part := range data {
 		encoded[offset] = part
 		offset++
 	}
-	data = varint.Pack64(uint64(testMeta.expires))
+	data = varint.Pack64(uint64(testMeta.Expires))
 	for _, part := range data {
 		encoded[offset] = part
 		offset++
 	}
-	data = varint.Pack64(uint64(testMeta.deleted))
+	data = varint.Pack64(uint64(testMeta.Deleted))
 	for _, part := range data {
 		encoded[offset] = part
 		offset++
@@ -254,7 +254,7 @@ func BenchmarkMetaUnserializeVarInt(b *testing.B) {
 			b.Error(err)
 			return
 		}
-		testMeta.created = int64(num)
+		testMeta.Created = int64(num)
 		offset += n
 
 		num, n, err = varint.Unpack64(encodedData[offset:])
@@ -262,7 +262,7 @@ func BenchmarkMetaUnserializeVarInt(b *testing.B) {
 			b.Error(err)
 			return
 		}
-		testMeta.modified = int64(num)
+		testMeta.Modified = int64(num)
 		offset += n
 
 		num, n, err = varint.Unpack64(encodedData[offset:])
@@ -270,7 +270,7 @@ func BenchmarkMetaUnserializeVarInt(b *testing.B) {
 			b.Error(err)
 			return
 		}
-		testMeta.expires = int64(num)
+		testMeta.Expires = int64(num)
 		offset += n
 
 		num, n, err = varint.Unpack64(encodedData[offset:])
@@ -278,7 +278,7 @@ func BenchmarkMetaUnserializeVarInt(b *testing.B) {
 			b.Error(err)
 			return
 		}
-		testMeta.deleted = int64(num)
+		testMeta.Deleted = int64(num)
 		offset += n
 
 		switch encodedData[offset] {
diff --git a/database/record/meta-gencode.go b/database/record/meta-gencode.go
index d79edff..c0a2142 100644
--- a/database/record/meta-gencode.go
+++ b/database/record/meta-gencode.go
@@ -33,78 +33,78 @@ func (d *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
 
 	{
 
-		buf[0+0] = byte(d.created >> 0)
+		buf[0+0] = byte(d.Created >> 0)
 
-		buf[1+0] = byte(d.created >> 8)
+		buf[1+0] = byte(d.Created >> 8)
 
-		buf[2+0] = byte(d.created >> 16)
+		buf[2+0] = byte(d.Created >> 16)
 
-		buf[3+0] = byte(d.created >> 24)
+		buf[3+0] = byte(d.Created >> 24)
 
-		buf[4+0] = byte(d.created >> 32)
+		buf[4+0] = byte(d.Created >> 32)
 
-		buf[5+0] = byte(d.created >> 40)
+		buf[5+0] = byte(d.Created >> 40)
 
-		buf[6+0] = byte(d.created >> 48)
+		buf[6+0] = byte(d.Created >> 48)
 
-		buf[7+0] = byte(d.created >> 56)
+		buf[7+0] = byte(d.Created >> 56)
 
 	}
 	{
 
-		buf[0+8] = byte(d.modified >> 0)
+		buf[0+8] = byte(d.Modified >> 0)
 
-		buf[1+8] = byte(d.modified >> 8)
+		buf[1+8] = byte(d.Modified >> 8)
 
-		buf[2+8] = byte(d.modified >> 16)
+		buf[2+8] = byte(d.Modified >> 16)
 
-		buf[3+8] = byte(d.modified >> 24)
+		buf[3+8] = byte(d.Modified >> 24)
 
-		buf[4+8] = byte(d.modified >> 32)
+		buf[4+8] = byte(d.Modified >> 32)
 
-		buf[5+8] = byte(d.modified >> 40)
+		buf[5+8] = byte(d.Modified >> 40)
 
-		buf[6+8] = byte(d.modified >> 48)
+		buf[6+8] = byte(d.Modified >> 48)
 
-		buf[7+8] = byte(d.modified >> 56)
+		buf[7+8] = byte(d.Modified >> 56)
 
 	}
 	{
 
-		buf[0+16] = byte(d.expires >> 0)
+		buf[0+16] = byte(d.Expires >> 0)
 
-		buf[1+16] = byte(d.expires >> 8)
+		buf[1+16] = byte(d.Expires >> 8)
 
-		buf[2+16] = byte(d.expires >> 16)
+		buf[2+16] = byte(d.Expires >> 16)
 
-		buf[3+16] = byte(d.expires >> 24)
+		buf[3+16] = byte(d.Expires >> 24)
 
-		buf[4+16] = byte(d.expires >> 32)
+		buf[4+16] = byte(d.Expires >> 32)
 
-		buf[5+16] = byte(d.expires >> 40)
+		buf[5+16] = byte(d.Expires >> 40)
 
-		buf[6+16] = byte(d.expires >> 48)
+		buf[6+16] = byte(d.Expires >> 48)
 
-		buf[7+16] = byte(d.expires >> 56)
+		buf[7+16] = byte(d.Expires >> 56)
 
 	}
 	{
 
-		buf[0+24] = byte(d.deleted >> 0)
+		buf[0+24] = byte(d.Deleted >> 0)
 
-		buf[1+24] = byte(d.deleted >> 8)
+		buf[1+24] = byte(d.Deleted >> 8)
 
-		buf[2+24] = byte(d.deleted >> 16)
+		buf[2+24] = byte(d.Deleted >> 16)
 
-		buf[3+24] = byte(d.deleted >> 24)
+		buf[3+24] = byte(d.Deleted >> 24)
 
-		buf[4+24] = byte(d.deleted >> 32)
+		buf[4+24] = byte(d.Deleted >> 32)
 
-		buf[5+24] = byte(d.deleted >> 40)
+		buf[5+24] = byte(d.Deleted >> 40)
 
-		buf[6+24] = byte(d.deleted >> 48)
+		buf[6+24] = byte(d.Deleted >> 48)
 
-		buf[7+24] = byte(d.deleted >> 56)
+		buf[7+24] = byte(d.Deleted >> 56)
 
 	}
 	{
@@ -134,22 +134,22 @@ func (d *Meta) GenCodeUnmarshal(buf []byte) (uint64, error) {
 
 	{
 
-		d.created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
+		d.Created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
 
 	}
 	{
 
-		d.modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
+		d.Modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
 
 	}
 	{
 
-		d.expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
+		d.Expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
 
 	}
 	{
 
-		d.deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
+		d.Deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
 
 	}
 	{
diff --git a/database/record/meta-gencode_test.go b/database/record/meta-gencode_test.go
index baea558..7050e7d 100644
--- a/database/record/meta-gencode_test.go
+++ b/database/record/meta-gencode_test.go
@@ -8,10 +8,10 @@ import (
 
 var (
 	genCodeTestMeta = &Meta{
-		created:   time.Now().Unix(),
-		modified:  time.Now().Unix(),
-		expires:   time.Now().Unix(),
-		deleted:   time.Now().Unix(),
+		Created:   time.Now().Unix(),
+		Modified:  time.Now().Unix(),
+		Expires:   time.Now().Unix(),
+		Deleted:   time.Now().Unix(),
 		secret:    true,
 		cronjewel: true,
 	}
diff --git a/database/record/meta.go b/database/record/meta.go
index c561a17..8ef2810 100644
--- a/database/record/meta.go
+++ b/database/record/meta.go
@@ -4,39 +4,39 @@ import "time"
 
 // Meta holds
 type Meta struct {
-	created   int64
-	modified  int64
-	expires   int64
-	deleted   int64
+	Created   int64
+	Modified  int64
+	Expires   int64
+	Deleted   int64
 	secret    bool // secrets must not be sent to the UI, only synced between nodes
 	cronjewel bool // crownjewels must never leave the instance, but may be read by the UI
 }
 
 // SetAbsoluteExpiry sets an absolute expiry time (in seconds), that is not affected when the record is updated.
 func (m *Meta) SetAbsoluteExpiry(seconds int64) {
-	m.expires = seconds
-	m.deleted = 0
+	m.Expires = seconds
+	m.Deleted = 0
 }
 
 // SetRelativateExpiry sets a relative expiry time (ie. TTL in seconds) that is automatically updated whenever the record is updated/saved.
 func (m *Meta) SetRelativateExpiry(seconds int64) {
 	if seconds >= 0 {
-		m.deleted = -seconds
+		m.Deleted = -seconds
 	}
 }
 
 // GetAbsoluteExpiry returns the absolute expiry time.
 func (m *Meta) GetAbsoluteExpiry() int64 {
-	return m.expires
+	return m.Expires
 }
 
 // GetRelativeExpiry returns the current relative expiry time - ie. seconds until expiry.
 func (m *Meta) GetRelativeExpiry() int64 {
-	if m.deleted < 0 {
-		return -m.deleted
+	if m.Deleted < 0 {
+		return -m.Deleted
 	}
 
-	abs := m.expires - time.Now().Unix()
+	abs := m.Expires - time.Now().Unix()
 	if abs < 0 {
 		return 0
 	}
@@ -56,30 +56,38 @@ func (m *Meta) MakeSecret() {
 // Update updates the internal meta states and should be called before writing the record to the database.
 func (m *Meta) Update() {
 	now := time.Now().Unix()
-	m.modified = now
-	if m.created == 0 {
-		m.created = now
+	m.Modified = now
+	if m.Created == 0 {
+		m.Created = now
 	}
-	if m.deleted < 0 {
-		m.expires = now - m.deleted
+	if m.Deleted < 0 {
+		m.Expires = now - m.Deleted
 	}
 }
 
 // Reset resets all metadata, except for the secret and crownjewel status.
 func (m *Meta) Reset() {
-	m.created = 0
-	m.modified = 0
-	m.expires = 0
-	m.deleted = 0
+	m.Created = 0
+	m.Modified = 0
+	m.Expires = 0
+	m.Deleted = 0
 }
 
-// CheckScope checks whether the current database record exists for the given scope.
-func (m *Meta) CheckScope(now int64, local, internal bool) (recordExists bool) {
+// CheckValidity checks whether the database record is valid.
+func (m *Meta) CheckValidity(now int64) (valid bool) {
 	switch {
-	case m.deleted > 0:
+	case m.Deleted > 0:
 		return false
-	case m.expires < now:
+	case m.Expires < now:
 		return false
+	default:
+		return true
+	}
+}
+
+// CheckPermission checks whether the database record may be accessed with the following scope.
+func (m *Meta) CheckPermission(local, internal bool) (permitted bool) {
+	switch {
 	case !local && m.cronjewel:
 		return false
 	case !internal && m.secret:
diff --git a/database/record/record.go b/database/record/record.go
index c002f25..c40fca2 100644
--- a/database/record/record.go
+++ b/database/record/record.go
@@ -16,4 +16,6 @@ type Record interface {
 
 	Lock()
 	Unlock()
+
+	IsWrapped() bool
 }
diff --git a/database/record/wrapper.go b/database/record/wrapper.go
index 3f73263..63b0367 100644
--- a/database/record/wrapper.go
+++ b/database/record/wrapper.go
@@ -118,3 +118,25 @@ func (w *Wrapper) Lock() {
 func (w *Wrapper) Unlock() {
 	w.lock.Unlock()
 }
+
+// IsWrapped returns whether the record is a Wrapper.
+func (w *Wrapper) IsWrapped() bool {
+	return true
+}
+
+func Unwrap(wrapped, new Record) (Record, error) {
+	wrapper, ok := wrapped.(*Wrapper)
+	if !ok {
+		return nil, fmt.Errorf("cannot unwrap %T", wrapped)
+	}
+
+	_, err := dsd.Load(wrapper.Data, new)
+	if err != nil {
+		return nil, fmt.Errorf("database: failed to unwrap %T: %s", new, err)
+	}
+
+	new.SetKey(wrapped.Key())
+	new.SetMeta(wrapped.Meta())
+
+	return new, nil
+}
diff --git a/database/registry.go b/database/registry.go
new file mode 100644
index 0000000..94734af
--- /dev/null
+++ b/database/registry.go
@@ -0,0 +1,145 @@
+package database
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sync"
+
+	"github.com/tevino/abool"
+)
+
+// RegisteredDatabase holds information about registered databases
+type RegisteredDatabase struct {
+	Name        string
+	Description string
+	StorageType string
+	PrimaryAPI  string
+}
+
+// Equal returns whether this instance equals another.
+func (r *RegisteredDatabase) Equal(o *RegisteredDatabase) bool {
+	if r.Name != o.Name ||
+		r.Description != o.Description ||
+		r.StorageType != o.StorageType ||
+		r.PrimaryAPI != o.PrimaryAPI {
+		return false
+	}
+	return true
+}
+
+const (
+	registryFileName = "databases.json"
+)
+
+var (
+	initialized = abool.NewBool(false)
+
+	registry     map[string]*RegisteredDatabase
+	registryLock sync.Mutex
+)
+
+// RegisterDatabase registers a new database.
+func RegisterDatabase(new *RegisteredDatabase) error {
+	if !initialized.IsSet() {
+		return errors.New("database not initialized")
+	}
+
+	registryLock.Lock()
+	defer registryLock.Unlock()
+
+	registeredDB, ok := registry[new.Name]
+	if !ok || !new.Equal(registeredDB) {
+		registry[new.Name] = new
+		return saveRegistry()
+	}
+
+	return nil
+}
+
+// Initialize initialized the database
+func Initialize(location string) error {
+	if initialized.SetToIf(false, true) {
+		rootDir = location
+
+		err := checkRootDir()
+		if err != nil {
+			return fmt.Errorf("could not create/open database directory (%s): %s", rootDir, err)
+		}
+
+		err = loadRegistry()
+		if err != nil {
+			return fmt.Errorf("could not load database registry (%s): %s", path.Join(rootDir, registryFileName), err)
+		}
+
+		return nil
+	}
+	return errors.New("database already initialized")
+}
+
+func checkRootDir() error {
+	// open dir
+	dir, err := os.Open(rootDir)
+	if err != nil {
+		if err == os.ErrNotExist {
+			return os.MkdirAll(rootDir, 0700)
+		}
+		return err
+	}
+	defer dir.Close()
+
+	fileInfo, err := dir.Stat()
+	if err != nil {
+		return err
+	}
+
+	if fileInfo.Mode().Perm() != 0700 {
+		return dir.Chmod(0700)
+	}
+	return nil
+}
+
+func loadRegistry() error {
+	registryLock.Lock()
+	defer registryLock.Unlock()
+
+	// read file
+	filePath := path.Join(rootDir, registryFileName)
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		if err == os.ErrNotExist {
+			registry = make(map[string]*RegisteredDatabase)
+			return nil
+		}
+		return err
+	}
+
+	// parse
+	new := make(map[string]*RegisteredDatabase)
+	err = json.Unmarshal(data, new)
+	if err != nil {
+		return err
+	}
+
+	// set
+	registry = new
+	return nil
+}
+
+func saveRegistry() error {
+	registryLock.Lock()
+	defer registryLock.Unlock()
+
+	// marshal
+	data, err := json.Marshal(registry)
+	if err != nil {
+		return err
+	}
+
+	// write file
+	filePath := path.Join(rootDir, registryFileName)
+	return ioutil.WriteFile(filePath, data, 0600)
+}
diff --git a/database/storage/interface.go b/database/storage/interface.go
index ae19a90..3832ca4 100644
--- a/database/storage/interface.go
+++ b/database/storage/interface.go
@@ -2,13 +2,12 @@ package storage
 
 import (
 	"github.com/Safing/portbase/database/iterator"
-	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/record"
 )
 
 // Interface defines the database storage API.
 type Interface interface {
-	Exists(key string) (bool, error)
 	Get(key string) (record.Record, error)
 	Put(m record.Record) error
 	Delete(key string) error
diff --git a/database/storage/storages.go b/database/storage/storages.go
index f4a3c27..948af83 100644
--- a/database/storage/storages.go
+++ b/database/storage/storages.go
@@ -28,6 +28,11 @@ func Register(name string, factory Factory) error {
 	return nil
 }
 
+// CreateDatabase starts a new database with the given name and storageType at location.
+func CreateDatabase(name, storageType, location string) (Interface, error) {
+	return nil, nil
+}
+
 // StartDatabase starts a new database with the given name and storageType at location.
 func StartDatabase(name, storageType, location string) (Interface, error) {
 	storagesLock.Lock()
diff --git a/database/subscriptions.go b/database/subscriptions.go
index 684016b..460713e 100644
--- a/database/subscriptions.go
+++ b/database/subscriptions.go
@@ -2,271 +2,271 @@
 
 package database
 
-import (
-	"fmt"
-	"strings"
-	"sync"
-
-	"github.com/Safing/safing-core/modules"
-	"github.com/Safing/safing-core/taskmanager"
-
-	"github.com/ipfs/go-datastore"
-	"github.com/tevino/abool"
-)
-
-var subscriptionModule *modules.Module
-var subscriptions []*Subscription
-var subLock sync.Mutex
-
-var databaseUpdate chan Model
-var databaseCreate chan Model
-var databaseDelete chan *datastore.Key
-
-var workIsWaiting chan *struct{}
-var workIsWaitingFlag *abool.AtomicBool
-var forceProcessing chan *struct{}
-
-type Subscription struct {
-	typeAndLocation map[string]bool
-	exactObject     map[string]bool
-	children        map[string]uint8
-	Created         chan Model
-	Updated         chan Model
-	Deleted         chan *datastore.Key
-}
-
-func NewSubscription() *Subscription {
-	subLock.Lock()
-	defer subLock.Unlock()
-	sub := &Subscription{
-		typeAndLocation: make(map[string]bool),
-		exactObject:     make(map[string]bool),
-		children:        make(map[string]uint8),
-		Created:         make(chan Model, 128),
-		Updated:         make(chan Model, 128),
-		Deleted:         make(chan *datastore.Key, 128),
-	}
-	subscriptions = append(subscriptions, sub)
-	return sub
-}
-
-func (sub *Subscription) Subscribe(subKey string) {
-	subLock.Lock()
-	defer subLock.Unlock()
-
-	namespaces := strings.Split(subKey, "/")[1:]
-	lastSpace := ""
-	if len(namespaces) != 0 {
-		lastSpace = namespaces[len(namespaces)-1]
-	}
-
-	switch {
-	case lastSpace == "":
-		// save key without leading "/"
-		// save with depth 255 to get all
-		sub.children[strings.Trim(subKey, "/")] = 0xFF
-	case strings.HasPrefix(lastSpace, "*"):
-		// save key without leading or trailing "/" or "*"
-		// save full wanted depth - this makes comparison easier
-		sub.children[strings.Trim(subKey, "/*")] = uint8(len(lastSpace) + len(namespaces) - 1)
-	case strings.Contains(lastSpace, ":"):
-		sub.exactObject[subKey] = true
-	default:
-		sub.typeAndLocation[subKey] = true
-	}
-}
-
-func (sub *Subscription) Unsubscribe(subKey string) {
-	subLock.Lock()
-	defer subLock.Unlock()
-
-	namespaces := strings.Split(subKey, "/")[1:]
-	lastSpace := ""
-	if len(namespaces) != 0 {
-		lastSpace = namespaces[len(namespaces)-1]
-	}
-
-	switch {
-	case lastSpace == "":
-		delete(sub.children, strings.Trim(subKey, "/"))
-	case strings.HasPrefix(lastSpace, "*"):
-		delete(sub.children, strings.Trim(subKey, "/*"))
-	case strings.Contains(lastSpace, ":"):
-		delete(sub.exactObject, subKey)
-	default:
-		delete(sub.typeAndLocation, subKey)
-	}
-}
-
-func (sub *Subscription) Destroy() {
-	subLock.Lock()
-	defer subLock.Unlock()
-
-	for k, v := range subscriptions {
-		if v.Created == sub.Created {
-			defer func() {
-				subscriptions = append(subscriptions[:k], subscriptions[k+1:]...)
-			}()
-			close(sub.Created)
-			close(sub.Updated)
-			close(sub.Deleted)
-			return
-		}
-	}
-}
-
-func (sub *Subscription) Subscriptions() *[]string {
-	subStrings := make([]string, 0)
-	for subString := range sub.exactObject {
-		subStrings = append(subStrings, subString)
-	}
-	for subString := range sub.typeAndLocation {
-		subStrings = append(subStrings, subString)
-	}
-	for subString, depth := range sub.children {
-		if depth == 0xFF {
-			subStrings = append(subStrings, fmt.Sprintf("/%s/", subString))
-		} else {
-			subStrings = append(subStrings, fmt.Sprintf("/%s/%s", subString, strings.Repeat("*", int(depth)-len(strings.Split(subString, "/")))))
-		}
-	}
-	return &subStrings
-}
-
-func (sub *Subscription) String() string {
-	return fmt.Sprintf("<Subscription [%s]>", strings.Join(*sub.Subscriptions(), " "))
-}
-
-func (sub *Subscription) send(key *datastore.Key, model Model, created bool) {
-	if model == nil {
-		sub.Deleted <- key
-	} else if created {
-		sub.Created <- model
-	} else {
-		sub.Updated <- model
-	}
-}
-
-func process(key *datastore.Key, model Model, created bool) {
-	subLock.Lock()
-	defer subLock.Unlock()
-
-	stringRep := key.String()
-	// "/Comedy/MontyPython/Actor:JohnCleese"
-	typeAndLocation := key.Path().String()
-	// "/Comedy/MontyPython/Actor"
-	namespaces := key.Namespaces()
-	// ["Comedy", "MontyPython", "Actor:JohnCleese"]
-	depth := uint8(len(namespaces))
-	// 3
-
-subscriptionLoop:
-	for _, sub := range subscriptions {
-		if _, ok := sub.exactObject[stringRep]; ok {
-			sub.send(key, model, created)
-			continue subscriptionLoop
-		}
-		if _, ok := sub.typeAndLocation[typeAndLocation]; ok {
-			sub.send(key, model, created)
-			continue subscriptionLoop
-		}
-		for i := 0; i < len(namespaces); i++ {
-			if subscribedDepth, ok := sub.children[strings.Join(namespaces[:i], "/")]; ok {
-				if subscribedDepth >= depth {
-					sub.send(key, model, created)
-					continue subscriptionLoop
-				}
-			}
-		}
-	}
-
-}
-
-func init() {
-	subscriptionModule = modules.Register("Database:Subscriptions", 128)
-	subscriptions = make([]*Subscription, 0)
-	subLock = sync.Mutex{}
-
-	databaseUpdate = make(chan Model, 32)
-	databaseCreate = make(chan Model, 32)
-	databaseDelete = make(chan *datastore.Key, 32)
-
-	workIsWaiting = make(chan *struct{}, 0)
-	workIsWaitingFlag = abool.NewBool(false)
-	forceProcessing = make(chan *struct{}, 0)
-
-	go run()
-}
-
-func run() {
-	for {
-		select {
-		case <-subscriptionModule.Stop:
-			subscriptionModule.StopComplete()
-			return
-		case <-workIsWaiting:
-			work()
-		}
-	}
-}
-
-func work() {
-	defer workIsWaitingFlag.UnSet()
-
-	// wait
-	select {
-	case <-taskmanager.StartMediumPriorityMicroTask():
-		defer taskmanager.EndMicroTask()
-	case <-forceProcessing:
-	}
-
-	// work
-	for {
-		select {
-		case model := <-databaseCreate:
-			process(model.GetKey(), model, true)
-		case model := <-databaseUpdate:
-			process(model.GetKey(), model, false)
-		case key := <-databaseDelete:
-			process(key, nil, false)
-		default:
-			return
-		}
-	}
-}
-
-func handleCreateSubscriptions(model Model) {
-	select {
-	case databaseCreate <- model:
-	default:
-		forceProcessing <- nil
-		databaseCreate <- model
-	}
-	if workIsWaitingFlag.SetToIf(false, true) {
-		workIsWaiting <- nil
-	}
-}
-
-func handleUpdateSubscriptions(model Model) {
-	select {
-	case databaseUpdate <- model:
-	default:
-		forceProcessing <- nil
-		databaseUpdate <- model
-	}
-	if workIsWaitingFlag.SetToIf(false, true) {
-		workIsWaiting <- nil
-	}
-}
-
-func handleDeleteSubscriptions(key *datastore.Key) {
-	select {
-	case databaseDelete <- key:
-	default:
-		forceProcessing <- nil
-		databaseDelete <- key
-	}
-	if workIsWaitingFlag.SetToIf(false, true) {
-		workIsWaiting <- nil
-	}
-}
+// import (
+// 	"fmt"
+// 	"strings"
+// 	"sync"
+//
+// 	"github.com/Safing/portbase/database/record"
+// 	"github.com/Safing/portbase/modules"
+// 	"github.com/Safing/portbase/taskmanager"
+//
+// 	"github.com/tevino/abool"
+// )
+//
+// var subscriptionModule *modules.Module
+// var subscriptions []*Subscription
+// var subLock sync.Mutex
+//
+// var databaseUpdate chan Model
+// var databaseCreate chan Model
+// var databaseDelete chan string
+//
+// var workIsWaiting chan *struct{}
+// var workIsWaitingFlag *abool.AtomicBool
+// var forceProcessing chan *struct{}
+//
+// type Subscription struct {
+// 	typeAndLocation map[string]bool
+// 	exactObject     map[string]bool
+// 	children        map[string]uint8
+// 	Created         chan record.Record
+// 	Updated         chan record.Record
+// 	Deleted         chan string
+// }
+//
+// func NewSubscription() *Subscription {
+// 	subLock.Lock()
+// 	defer subLock.Unlock()
+// 	sub := &Subscription{
+// 		typeAndLocation: make(map[string]bool),
+// 		exactObject:     make(map[string]bool),
+// 		children:        make(map[string]uint8),
+// 		Created:         make(chan record.Record, 128),
+// 		Updated:         make(chan record.Record, 128),
+// 		Deleted:         make(chan string, 128),
+// 	}
+// 	subscriptions = append(subscriptions, sub)
+// 	return sub
+// }
+//
+// func (sub *Subscription) Subscribe(subKey string) {
+// 	subLock.Lock()
+// 	defer subLock.Unlock()
+//
+// 	namespaces := strings.Split(subKey, "/")[1:]
+// 	lastSpace := ""
+// 	if len(namespaces) != 0 {
+// 		lastSpace = namespaces[len(namespaces)-1]
+// 	}
+//
+// 	switch {
+// 	case lastSpace == "":
+// 		// save key without leading "/"
+// 		// save with depth 255 to get all
+// 		sub.children[strings.Trim(subKey, "/")] = 0xFF
+// 	case strings.HasPrefix(lastSpace, "*"):
+// 		// save key without leading or trailing "/" or "*"
+// 		// save full wanted depth - this makes comparison easier
+// 		sub.children[strings.Trim(subKey, "/*")] = uint8(len(lastSpace) + len(namespaces) - 1)
+// 	case strings.Contains(lastSpace, ":"):
+// 		sub.exactObject[subKey] = true
+// 	default:
+// 		sub.typeAndLocation[subKey] = true
+// 	}
+// }
+//
+// func (sub *Subscription) Unsubscribe(subKey string) {
+// 	subLock.Lock()
+// 	defer subLock.Unlock()
+//
+// 	namespaces := strings.Split(subKey, "/")[1:]
+// 	lastSpace := ""
+// 	if len(namespaces) != 0 {
+// 		lastSpace = namespaces[len(namespaces)-1]
+// 	}
+//
+// 	switch {
+// 	case lastSpace == "":
+// 		delete(sub.children, strings.Trim(subKey, "/"))
+// 	case strings.HasPrefix(lastSpace, "*"):
+// 		delete(sub.children, strings.Trim(subKey, "/*"))
+// 	case strings.Contains(lastSpace, ":"):
+// 		delete(sub.exactObject, subKey)
+// 	default:
+// 		delete(sub.typeAndLocation, subKey)
+// 	}
+// }
+//
+// func (sub *Subscription) Destroy() {
+// 	subLock.Lock()
+// 	defer subLock.Unlock()
+//
+// 	for k, v := range subscriptions {
+// 		if v.Created == sub.Created {
+// 			defer func() {
+// 				subscriptions = append(subscriptions[:k], subscriptions[k+1:]...)
+// 			}()
+// 			close(sub.Created)
+// 			close(sub.Updated)
+// 			close(sub.Deleted)
+// 			return
+// 		}
+// 	}
+// }
+//
+// func (sub *Subscription) Subscriptions() *[]string {
+// 	subStrings := make([]string, 0)
+// 	for subString := range sub.exactObject {
+// 		subStrings = append(subStrings, subString)
+// 	}
+// 	for subString := range sub.typeAndLocation {
+// 		subStrings = append(subStrings, subString)
+// 	}
+// 	for subString, depth := range sub.children {
+// 		if depth == 0xFF {
+// 			subStrings = append(subStrings, fmt.Sprintf("/%s/", subString))
+// 		} else {
+// 			subStrings = append(subStrings, fmt.Sprintf("/%s/%s", subString, strings.Repeat("*", int(depth)-len(strings.Split(subString, "/")))))
+// 		}
+// 	}
+// 	return &subStrings
+// }
+//
+// func (sub *Subscription) String() string {
+// 	return fmt.Sprintf("<Subscription [%s]>", strings.Join(*sub.Subscriptions(), " "))
+// }
+//
+// func (sub *Subscription) send(key string, rec record.Record, created bool) {
+// 	if rec == nil {
+// 		sub.Deleted <- key
+// 	} else if created {
+// 		sub.Created <- rec
+// 	} else {
+// 		sub.Updated <- rec
+// 	}
+// }
+//
+// func process(key string, rec record.Record, created bool) {
+// 	subLock.Lock()
+// 	defer subLock.Unlock()
+//
+// 	stringRep := key.String()
+// 	// "/Comedy/MontyPython/Actor:JohnCleese"
+// 	typeAndLocation := key.Path().String()
+// 	// "/Comedy/MontyPython/Actor"
+// 	namespaces := key.Namespaces()
+// 	// ["Comedy", "MontyPython", "Actor:JohnCleese"]
+// 	depth := uint8(len(namespaces))
+// 	// 3
+//
+// subscriptionLoop:
+// 	for _, sub := range subscriptions {
+// 		if _, ok := sub.exactObject[stringRep]; ok {
+// 			sub.send(key, rec, created)
+// 			continue subscriptionLoop
+// 		}
+// 		if _, ok := sub.typeAndLocation[typeAndLocation]; ok {
+// 			sub.send(key, rec, created)
+// 			continue subscriptionLoop
+// 		}
+// 		for i := 0; i < len(namespaces); i++ {
+// 			if subscribedDepth, ok := sub.children[strings.Join(namespaces[:i], "/")]; ok {
+// 				if subscribedDepth >= depth {
+// 					sub.send(key, rec, created)
+// 					continue subscriptionLoop
+// 				}
+// 			}
+// 		}
+// 	}
+//
+// }
+//
+// func init() {
+// 	subscriptionModule = modules.Register("Database:Subscriptions", 128)
+// 	subscriptions = make([]*Subscription, 0)
+// 	subLock = sync.Mutex{}
+//
+// 	databaseUpdate = make(chan Model, 32)
+// 	databaseCreate = make(chan Model, 32)
+// 	databaseDelete = make(chan string, 32)
+//
+// 	workIsWaiting = make(chan *struct{}, 0)
+// 	workIsWaitingFlag = abool.NewBool(false)
+// 	forceProcessing = make(chan *struct{}, 0)
+//
+// 	go run()
+// }
+//
+// func run() {
+// 	for {
+// 		select {
+// 		case <-subscriptionModule.Stop:
+// 			subscriptionModule.StopComplete()
+// 			return
+// 		case <-workIsWaiting:
+// 			work()
+// 		}
+// 	}
+// }
+//
+// func work() {
+// 	defer workIsWaitingFlag.UnSet()
+//
+// 	// wait
+// 	select {
+// 	case <-taskmanager.StartMediumPriorityMicroTask():
+// 		defer taskmanager.EndMicroTask()
+// 	case <-forceProcessing:
+// 	}
+//
+// 	// work
+// 	for {
+// 		select {
+// 		case rec := <-databaseCreate:
+// 			process(rec.GetKey(), rec, true)
+// 		case rec := <-databaseUpdate:
+// 			process(rec.GetKey(), rec, false)
+// 		case key := <-databaseDelete:
+// 			process(key, nil, false)
+// 		default:
+// 			return
+// 		}
+// 	}
+// }
+//
+// func handleCreateSubscriptions(rec record.Record) {
+// 	select {
+// 	case databaseCreate <- rec:
+// 	default:
+// 		forceProcessing <- nil
+// 		databaseCreate <- rec
+// 	}
+// 	if workIsWaitingFlag.SetToIf(false, true) {
+// 		workIsWaiting <- nil
+// 	}
+// }
+//
+// func handleUpdateSubscriptions(rec record.Record) {
+// 	select {
+// 	case databaseUpdate <- rec:
+// 	default:
+// 		forceProcessing <- nil
+// 		databaseUpdate <- rec
+// 	}
+// 	if workIsWaitingFlag.SetToIf(false, true) {
+// 		workIsWaiting <- nil
+// 	}
+// }
+//
+// func handleDeleteSubscriptions(key string) {
+// 	select {
+// 	case databaseDelete <- key:
+// 	default:
+// 		forceProcessing <- nil
+// 		databaseDelete <- key
+// 	}
+// 	if workIsWaitingFlag.SetToIf(false, true) {
+// 		workIsWaiting <- nil
+// 	}
+// }
diff --git a/database/subscriptions_test.go b/database/subscriptions_test.go
index 817a9df..5ab06d3 100644
--- a/database/subscriptions_test.go
+++ b/database/subscriptions_test.go
@@ -2,102 +2,102 @@
 
 package database
 
-import (
-	"strconv"
-	"strings"
-	"sync"
-	"testing"
-)
-
-var subTestWg sync.WaitGroup
-
-func waitForSubs(t *testing.T, sub *Subscription, highest int) {
-	defer subTestWg.Done()
-	expecting := 1
-	var subbedModel Model
-forLoop:
-	for {
-		select {
-		case subbedModel = <-sub.Created:
-		case subbedModel = <-sub.Updated:
-		}
-		t.Logf("got model from subscription: %s", subbedModel.GetKey().String())
-		if !strings.HasPrefix(subbedModel.GetKey().Name(), "sub") {
-			// not a model that we use for testing, other tests might be interfering
-			continue forLoop
-		}
-		number, err := strconv.Atoi(strings.TrimPrefix(subbedModel.GetKey().Name(), "sub"))
-		if err != nil || number != expecting {
-			t.Errorf("test subscription: got unexpected model %s, expected sub%d", subbedModel.GetKey().String(), expecting)
-			continue forLoop
-		}
-		if number == highest {
-			return
-		}
-		expecting++
-	}
-}
-
-func TestSubscriptions(t *testing.T) {
-
-	// create subscription
-	sub := NewSubscription()
-
-	// FIRST TEST
-
-	subTestWg.Add(1)
-	go waitForSubs(t, sub, 3)
-	sub.Subscribe("/Tests/")
-	t.Log(sub.String())
-
-	(&(TestingModel{})).CreateInNamespace("", "sub1")
-	(&(TestingModel{})).CreateInNamespace("A", "sub2")
-	(&(TestingModel{})).CreateInNamespace("A/B/C/D/E", "sub3")
-
-	subTestWg.Wait()
-
-	// SECOND TEST
-
-	subTestWg.Add(1)
-	go waitForSubs(t, sub, 3)
-	sub.Unsubscribe("/Tests/")
-	sub.Subscribe("/Tests/A/****")
-	t.Log(sub.String())
-
-	(&(TestingModel{})).CreateInNamespace("", "subX")
-	(&(TestingModel{})).CreateInNamespace("A", "sub1")
-	(&(TestingModel{})).CreateInNamespace("A/B/C/D", "sub2")
-	(&(TestingModel{})).CreateInNamespace("A/B/C/D/E", "subX")
-	(&(TestingModel{})).CreateInNamespace("A", "sub3")
-
-	subTestWg.Wait()
-
-	// THIRD TEST
-
-	subTestWg.Add(1)
-	go waitForSubs(t, sub, 3)
-	sub.Unsubscribe("/Tests/A/****")
-	sub.Subscribe("/Tests/TestingModel:sub1")
-	sub.Subscribe("/Tests/TestingModel:sub1/TestingModel")
-	t.Log(sub.String())
-
-	(&(TestingModel{})).CreateInNamespace("", "sub1")
-	(&(TestingModel{})).CreateInNamespace("", "subX")
-	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1", "sub2")
-	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1/A", "subX")
-	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1", "sub3")
-
-	subTestWg.Wait()
-
-	// FINAL STUFF
-
-	model := &TestingModel{}
-	model.CreateInNamespace("Invalid", "subX")
-	model.Save()
-
-	sub.Destroy()
-
-	// time.Sleep(1 * time.Second)
-	// pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
-
-}
+// import (
+// 	"strconv"
+// 	"strings"
+// 	"sync"
+// 	"testing"
+// )
+//
+// var subTestWg sync.WaitGroup
+//
+// func waitForSubs(t *testing.T, sub *Subscription, highest int) {
+// 	defer subTestWg.Done()
+// 	expecting := 1
+// 	var subbedModel Model
+// forLoop:
+// 	for {
+// 		select {
+// 		case subbedModel = <-sub.Created:
+// 		case subbedModel = <-sub.Updated:
+// 		}
+// 		t.Logf("got model from subscription: %s", subbedModel.GetKey().String())
+// 		if !strings.HasPrefix(subbedModel.GetKey().Name(), "sub") {
+// 			// not a model that we use for testing, other tests might be interfering
+// 			continue forLoop
+// 		}
+// 		number, err := strconv.Atoi(strings.TrimPrefix(subbedModel.GetKey().Name(), "sub"))
+// 		if err != nil || number != expecting {
+// 			t.Errorf("test subscription: got unexpected model %s, expected sub%d", subbedModel.GetKey().String(), expecting)
+// 			continue forLoop
+// 		}
+// 		if number == highest {
+// 			return
+// 		}
+// 		expecting++
+// 	}
+// }
+//
+// func TestSubscriptions(t *testing.T) {
+//
+// 	// create subscription
+// 	sub := NewSubscription()
+//
+// 	// FIRST TEST
+//
+// 	subTestWg.Add(1)
+// 	go waitForSubs(t, sub, 3)
+// 	sub.Subscribe("/Tests/")
+// 	t.Log(sub.String())
+//
+// 	(&(TestingModel{})).CreateInNamespace("", "sub1")
+// 	(&(TestingModel{})).CreateInNamespace("A", "sub2")
+// 	(&(TestingModel{})).CreateInNamespace("A/B/C/D/E", "sub3")
+//
+// 	subTestWg.Wait()
+//
+// 	// SECOND TEST
+//
+// 	subTestWg.Add(1)
+// 	go waitForSubs(t, sub, 3)
+// 	sub.Unsubscribe("/Tests/")
+// 	sub.Subscribe("/Tests/A/****")
+// 	t.Log(sub.String())
+//
+// 	(&(TestingModel{})).CreateInNamespace("", "subX")
+// 	(&(TestingModel{})).CreateInNamespace("A", "sub1")
+// 	(&(TestingModel{})).CreateInNamespace("A/B/C/D", "sub2")
+// 	(&(TestingModel{})).CreateInNamespace("A/B/C/D/E", "subX")
+// 	(&(TestingModel{})).CreateInNamespace("A", "sub3")
+//
+// 	subTestWg.Wait()
+//
+// 	// THIRD TEST
+//
+// 	subTestWg.Add(1)
+// 	go waitForSubs(t, sub, 3)
+// 	sub.Unsubscribe("/Tests/A/****")
+// 	sub.Subscribe("/Tests/TestingModel:sub1")
+// 	sub.Subscribe("/Tests/TestingModel:sub1/TestingModel")
+// 	t.Log(sub.String())
+//
+// 	(&(TestingModel{})).CreateInNamespace("", "sub1")
+// 	(&(TestingModel{})).CreateInNamespace("", "subX")
+// 	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1", "sub2")
+// 	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1/A", "subX")
+// 	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1", "sub3")
+//
+// 	subTestWg.Wait()
+//
+// 	// FINAL STUFF
+//
+// 	model := &TestingModel{}
+// 	model.CreateInNamespace("Invalid", "subX")
+// 	model.Save()
+//
+// 	sub.Destroy()
+//
+// 	// time.Sleep(1 * time.Second)
+// 	// pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+//
+// }
diff --git a/database/wrapper.go b/database/wrapper.go
deleted file mode 100644
index 74d3089..0000000
--- a/database/wrapper.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"github.com/ipfs/go-datastore"
-
-	"github.com/Safing/safing-core/database/dbutils"
-)
-
-func NewWrapper(key *datastore.Key, data []byte) (*dbutils.Wrapper, error) {
-	return dbutils.NewWrapper(key, data)
-}
-
-func DumpModel(uncertain interface{}, storageType uint8) ([]byte, error) {
-	return dbutils.DumpModel(uncertain, storageType)
-}
diff --git a/database/wrapper_test.go b/database/wrapper_test.go
deleted file mode 100644
index 27ce5f2..0000000
--- a/database/wrapper_test.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"testing"
-
-	"github.com/Safing/safing-core/formats/dsd"
-)
-
-func TestWrapper(t *testing.T) {
-
-	// create Model
-	new := &TestingModel{
-		Name:  "a",
-		Value: "b",
-	}
-	newTwo := &TestingModel{
-		Name:  "c",
-		Value: "d",
-	}
-
-	// dump
-	bytes, err := DumpModel(new, dsd.JSON)
-	if err != nil {
-		panic(err)
-	}
-	bytesTwo, err := DumpModel(newTwo, dsd.JSON)
-	if err != nil {
-		panic(err)
-	}
-
-	// wrap
-	wrapped, err := NewWrapper(nil, bytes)
-	if err != nil {
-		panic(err)
-	}
-	wrappedTwo, err := NewWrapper(nil, bytesTwo)
-	if err != nil {
-		panic(err)
-	}
-
-	// model definition for unwrapping
-	var model *TestingModel
-
-	// unwrap
-	myModel, ok := SilentEnsureModel(wrapped, model).(*TestingModel)
-	if !ok {
-		panic("received model does not match expected model")
-	}
-	if myModel.Name != "a" || myModel.Value != "b" {
-		panic("model value mismatch")
-	}
-
-	// verbose unwrap
-	genericModel, err := EnsureModel(wrappedTwo, model)
-	if err != nil {
-		panic(err)
-	}
-	myModelTwo, ok := genericModel.(*TestingModel)
-	if !ok {
-		panic("received model does not match expected model")
-	}
-	if myModelTwo.Name != "c" || myModelTwo.Value != "d" {
-		panic("model value mismatch")
-	}
-
-}

From 818cb332b437b251ef79dbf8ac39ab9dfd60ef1e Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 7 Sep 2018 19:14:58 +0200
Subject: [PATCH 13/20] Revamp feeder to accessor

---
 database/record/accessor-json-bytes.go  |  81 ++++++++++
 database/record/accessor-json-string.go |  81 ++++++++++
 database/record/accessor-struct.go      | 149 +++++++++++++++++++
 database/record/accessor.go             |  18 +++
 database/record/accessor_test.go        | 188 ++++++++++++++++++++++++
 database/record/base.go                 |   8 +
 database/record/wrapper.go              |   8 +
 database/record/wrapper_test.go         |   2 +-
 8 files changed, 534 insertions(+), 1 deletion(-)
 create mode 100644 database/record/accessor-json-bytes.go
 create mode 100644 database/record/accessor-json-string.go
 create mode 100644 database/record/accessor-struct.go
 create mode 100644 database/record/accessor.go
 create mode 100644 database/record/accessor_test.go

diff --git a/database/record/accessor-json-bytes.go b/database/record/accessor-json-bytes.go
new file mode 100644
index 0000000..ebb107f
--- /dev/null
+++ b/database/record/accessor-json-bytes.go
@@ -0,0 +1,81 @@
+package record
+
+import (
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+// JSONBytesAccessor is a json string with get functions.
+type JSONBytesAccessor struct {
+	json *[]byte
+}
+
+// NewJSONBytesAccessor adds the Accessor interface to a JSON bytes string.
+func NewJSONBytesAccessor(json *[]byte) *JSONBytesAccessor {
+	return &JSONBytesAccessor{
+		json: json,
+	}
+}
+
+// Set sets the value identified by key.
+func (ja *JSONBytesAccessor) Set(key string, value interface{}) error {
+	new, err := sjson.SetBytes(*ja.json, key, value)
+	if err != nil {
+		return err
+	}
+	*ja.json = new
+	return nil
+}
+
+// GetString returns the string found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetString(key string) (value string, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.String {
+		return emptyString, false
+	}
+	return result.String(), true
+}
+
+// GetInt returns the int found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetInt(key string) (value int64, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Int(), true
+}
+
+// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetFloat(key string) (value float64, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Float(), true
+}
+
+// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetBool(key string) (value bool, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	switch {
+	case !result.Exists():
+		return false, false
+	case result.Type == gjson.True:
+		return true, true
+	case result.Type == gjson.False:
+		return false, true
+	default:
+		return false, false
+	}
+}
+
+// Exists returns the whether the given key exists.
+func (ja *JSONBytesAccessor) Exists(key string) bool {
+	result := gjson.GetBytes(*ja.json, key)
+	return result.Exists()
+}
+
+// Type returns the accessor type as a string.
+func (ja *JSONBytesAccessor) Type() string {
+	return "JSONBytesAccessor"
+}
diff --git a/database/record/accessor-json-string.go b/database/record/accessor-json-string.go
new file mode 100644
index 0000000..4a4bd61
--- /dev/null
+++ b/database/record/accessor-json-string.go
@@ -0,0 +1,81 @@
+package record
+
+import (
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+// JSONAccessor is a json string with get functions.
+type JSONAccessor struct {
+	json *string
+}
+
+// NewJSONAccessor adds the Accessor interface to a JSON string.
+func NewJSONAccessor(json *string) *JSONAccessor {
+	return &JSONAccessor{
+		json: json,
+	}
+}
+
+// Set sets the value identified by key.
+func (ja *JSONAccessor) Set(key string, value interface{}) error {
+	new, err := sjson.Set(*ja.json, key, value)
+	if err != nil {
+		return err
+	}
+	*ja.json = new
+	return nil
+}
+
+// GetString returns the string found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetString(key string) (value string, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.String {
+		return emptyString, false
+	}
+	return result.String(), true
+}
+
+// GetInt returns the int found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetInt(key string) (value int64, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Int(), true
+}
+
+// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetFloat(key string) (value float64, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Float(), true
+}
+
+// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetBool(key string) (value bool, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	switch {
+	case !result.Exists():
+		return false, false
+	case result.Type == gjson.True:
+		return true, true
+	case result.Type == gjson.False:
+		return false, true
+	default:
+		return false, false
+	}
+}
+
+// Exists returns the whether the given key exists.
+func (ja *JSONAccessor) Exists(key string) bool {
+	result := gjson.Get(*ja.json, key)
+	return result.Exists()
+}
+
+// Type returns the accessor type as a string.
+func (ja *JSONAccessor) Type() string {
+	return "JSONAccessor"
+}
diff --git a/database/record/accessor-struct.go b/database/record/accessor-struct.go
new file mode 100644
index 0000000..9cafc39
--- /dev/null
+++ b/database/record/accessor-struct.go
@@ -0,0 +1,149 @@
+package record
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+)
+
+// StructAccessor is a json string with get functions.
+type StructAccessor struct {
+	object reflect.Value
+}
+
+// NewStructAccessor adds the Accessor interface to a JSON string.
+func NewStructAccessor(object interface{}) *StructAccessor {
+	return &StructAccessor{
+		object: reflect.ValueOf(object).Elem(),
+	}
+}
+
+// Set sets the value identified by key.
+func (sa *StructAccessor) Set(key string, value interface{}) error {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() {
+		return errors.New("struct field does not exist")
+	}
+	if !field.CanSet() {
+		return fmt.Errorf("field %s or struct is immutable", field.String())
+	}
+
+	newVal := reflect.ValueOf(value)
+
+	// set directly if type matches
+	if newVal.Kind() == field.Kind() {
+		field.Set(newVal)
+		return nil
+	}
+
+	// handle special cases
+	switch field.Kind() {
+
+	// ints
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		var newInt int64
+		switch newVal.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			newInt = newVal.Int()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			newInt = int64(newVal.Uint())
+		default:
+			return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+		}
+		if field.OverflowInt(newInt) {
+			return fmt.Errorf("setting field %s (%s) to %d would overflow", key, field.Kind().String(), newInt)
+		}
+		field.SetInt(newInt)
+
+		// uints
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		var newUint uint64
+		switch newVal.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			newUint = uint64(newVal.Int())
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			newUint = newVal.Uint()
+		default:
+			return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+		}
+		if field.OverflowUint(newUint) {
+			return fmt.Errorf("setting field %s (%s) to %d would overflow", key, field.Kind().String(), newUint)
+		}
+		field.SetUint(newUint)
+
+		// floats
+	case reflect.Float32, reflect.Float64:
+		switch newVal.Kind() {
+		case reflect.Float32, reflect.Float64:
+			field.SetFloat(newVal.Float())
+		default:
+			return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+		}
+	default:
+		return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+	}
+
+	return nil
+}
+
+// GetString returns the string found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetString(key string) (value string, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() || field.Kind() != reflect.String {
+		return "", false
+	}
+	return field.String(), true
+}
+
+// GetInt returns the int found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetInt(key string) (value int64, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() {
+		return 0, false
+	}
+	switch field.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return field.Int(), true
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return int64(field.Uint()), true
+	default:
+		return 0, false
+	}
+}
+
+// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetFloat(key string) (value float64, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() {
+		return 0, false
+	}
+	switch field.Kind() {
+	case reflect.Float32, reflect.Float64:
+		return field.Float(), true
+	default:
+		return 0, false
+	}
+}
+
+// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetBool(key string) (value bool, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() || field.Kind() != reflect.Bool {
+		return false, false
+	}
+	return field.Bool(), true
+}
+
+// Exists returns the whether the given key exists.
+func (sa *StructAccessor) Exists(key string) bool {
+	field := sa.object.FieldByName(key)
+	if field.IsValid() {
+		return true
+	}
+	return false
+}
+
+// Type returns the accessor type as a string.
+func (sa *StructAccessor) Type() string {
+	return "StructAccessor"
+}
diff --git a/database/record/accessor.go b/database/record/accessor.go
new file mode 100644
index 0000000..d6856bc
--- /dev/null
+++ b/database/record/accessor.go
@@ -0,0 +1,18 @@
+package record
+
+const (
+	emptyString = ""
+)
+
+// Accessor provides an interface to supply the query matcher a method to retrieve values from an object.
+type Accessor interface {
+	GetString(key string) (value string, ok bool)
+	GetInt(key string) (value int64, ok bool)
+	GetFloat(key string) (value float64, ok bool)
+	GetBool(key string) (value bool, ok bool)
+	Exists(key string) bool
+
+	Set(key string, value interface{}) error
+
+	Type() string
+}
diff --git a/database/record/accessor_test.go b/database/record/accessor_test.go
new file mode 100644
index 0000000..12cbcac
--- /dev/null
+++ b/database/record/accessor_test.go
@@ -0,0 +1,188 @@
+package record
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+type TestStruct struct {
+	S    string
+	I    int
+	I8   int8
+	I16  int16
+	I32  int32
+	I64  int64
+	UI   uint
+	UI8  uint8
+	UI16 uint16
+	UI32 uint32
+	UI64 uint64
+	F32  float32
+	F64  float64
+	B    bool
+}
+
+var (
+	testStruct = &TestStruct{
+		S:    "banana",
+		I:    42,
+		I8:   42,
+		I16:  42,
+		I32:  42,
+		I64:  42,
+		UI:   42,
+		UI8:  42,
+		UI16: 42,
+		UI32: 42,
+		UI64: 42,
+		F32:  42.42,
+		F64:  42.42,
+		B:    true,
+	}
+	testJSONBytes, _ = json.Marshal(testStruct)
+	testJSON         = string(testJSONBytes)
+)
+
+func testGetString(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue string) {
+	v, ok := acc.GetString(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get string with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get string with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if v != expectedValue {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testGetInt(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue int64) {
+	v, ok := acc.GetInt(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get int with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get int with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if v != expectedValue {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testGetFloat(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue float64) {
+	v, ok := acc.GetFloat(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get float with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get float with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if int64(v) != int64(expectedValue) {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testGetBool(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue bool) {
+	v, ok := acc.GetBool(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get bool with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get bool with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if v != expectedValue {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testSet(t *testing.T, acc Accessor, key string, shouldSucceed bool, valueToSet interface{}) {
+	err := acc.Set(key, valueToSet)
+	switch {
+	case err != nil && shouldSucceed:
+		t.Errorf("%s failed to set %s to %+v: %s", acc.Type(), key, valueToSet, err)
+	case err == nil && !shouldSucceed:
+		t.Errorf("%s should have failed to set %s to %+v", acc.Type(), key, valueToSet)
+	}
+}
+
+func TestAccessor(t *testing.T) {
+
+	// Test interface compliance
+	accs := []Accessor{
+		NewJSONAccessor(&testJSON),
+		NewJSONBytesAccessor(&testJSONBytes),
+		NewStructAccessor(testStruct),
+	}
+
+	// get
+	for _, acc := range accs {
+		testGetString(t, acc, "S", true, "banana")
+		testGetInt(t, acc, "I", true, 42)
+		testGetInt(t, acc, "I8", true, 42)
+		testGetInt(t, acc, "I16", true, 42)
+		testGetInt(t, acc, "I32", true, 42)
+		testGetInt(t, acc, "I64", true, 42)
+		testGetInt(t, acc, "UI", true, 42)
+		testGetInt(t, acc, "UI8", true, 42)
+		testGetInt(t, acc, "UI16", true, 42)
+		testGetInt(t, acc, "UI32", true, 42)
+		testGetInt(t, acc, "UI64", true, 42)
+		testGetFloat(t, acc, "F32", true, 42.42)
+		testGetFloat(t, acc, "F64", true, 42.42)
+		testGetBool(t, acc, "B", true, true)
+	}
+
+	// set
+	for _, acc := range accs {
+		testSet(t, acc, "S", true, "coconut")
+		testSet(t, acc, "I", true, uint32(44))
+		testSet(t, acc, "I8", true, uint64(44))
+		testSet(t, acc, "I16", true, uint8(44))
+		testSet(t, acc, "I32", true, uint16(44))
+		testSet(t, acc, "I64", true, 44)
+		testSet(t, acc, "UI", true, 44)
+		testSet(t, acc, "UI8", true, int64(44))
+		testSet(t, acc, "UI16", true, int32(44))
+		testSet(t, acc, "UI32", true, int8(44))
+		testSet(t, acc, "UI64", true, int16(44))
+		testSet(t, acc, "F32", true, 44.44)
+		testSet(t, acc, "F64", true, 44.44)
+		testSet(t, acc, "B", true, false)
+	}
+
+	// get again
+	for _, acc := range accs {
+		testGetString(t, acc, "S", true, "coconut")
+		testGetInt(t, acc, "I", true, 44)
+		testGetInt(t, acc, "I8", true, 44)
+		testGetInt(t, acc, "I16", true, 44)
+		testGetInt(t, acc, "I32", true, 44)
+		testGetInt(t, acc, "I64", true, 44)
+		testGetInt(t, acc, "UI", true, 44)
+		testGetInt(t, acc, "UI8", true, 44)
+		testGetInt(t, acc, "UI16", true, 44)
+		testGetInt(t, acc, "UI32", true, 44)
+		testGetInt(t, acc, "UI64", true, 44)
+		testGetFloat(t, acc, "F32", true, 44.44)
+		testGetFloat(t, acc, "F64", true, 44.44)
+		testGetBool(t, acc, "B", true, false)
+	}
+
+	// failures
+	for _, acc := range accs {
+		testGetString(t, acc, "S", false, 1)
+		testGetInt(t, acc, "I", false, 44)
+		testGetInt(t, acc, "I8", false, 512)
+		testGetInt(t, acc, "I16", false, 1000000)
+		testGetInt(t, acc, "I32", false, 44)
+		testGetInt(t, acc, "I64", false, "44")
+		testGetInt(t, acc, "UI", false, 44)
+		testGetInt(t, acc, "UI8", false, 44)
+		testGetInt(t, acc, "UI16", false, 44)
+		testGetInt(t, acc, "UI32", false, 44)
+		testGetInt(t, acc, "UI64", false, 44)
+		testGetFloat(t, acc, "F32", false, 44.44)
+		testGetFloat(t, acc, "F64", false, 44.44)
+		testGetBool(t, acc, "B", false, false)
+	}
+}
diff --git a/database/record/base.go b/database/record/base.go
index d1e2b04..c2966bb 100644
--- a/database/record/base.go
+++ b/database/record/base.go
@@ -53,6 +53,14 @@ func (b *Base) SetMeta(meta *Meta) {
 
 // Marshal marshals the object, without the database key or metadata
 func (b *Base) Marshal(format uint8) ([]byte, error) {
+	if b.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if b.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
 	dumped, err := dsd.Dump(b, format)
 	if err != nil {
 		return nil, err
diff --git a/database/record/wrapper.go b/database/record/wrapper.go
index 63b0367..d6352b7 100644
--- a/database/record/wrapper.go
+++ b/database/record/wrapper.go
@@ -75,6 +75,14 @@ func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
 
 // Marshal marshals the object, without the database key or metadata
 func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if w.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
 	if storageType != dsd.AUTO && storageType != w.Format {
 		return nil, errors.New("could not dump model, wrapped object format mismatch")
 	}
diff --git a/database/record/wrapper_test.go b/database/record/wrapper_test.go
index 476ff2f..e725916 100644
--- a/database/record/wrapper_test.go
+++ b/database/record/wrapper_test.go
@@ -19,7 +19,7 @@ func TestWrapper(t *testing.T) {
 	testData := []byte(`J{"a": "b"}`)
 
 	// test wrapper
-	wrapper, err := NewWrapper("test:a", nil, testData)
+	wrapper, err := NewWrapper("test:a", &Meta{}, testData)
 	if err != nil {
 		t.Fatal(err)
 	}

From 3d604313768adab49a8fa8f2fb7a31afea6b2070 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 7 Sep 2018 19:15:40 +0200
Subject: [PATCH 14/20] Work on datbase controller

---
 database/controller.go                | 59 ++++++++++++++++++++
 database/database.go                  |  5 +-
 database/query/fetcher.go             | 78 ---------------------------
 database/storage/badger/badger.go     | 17 +++---
 database/storage/interface.go         |  1 +
 database/storage/sinkhole/sinkhole.go |  7 ++-
 6 files changed, 77 insertions(+), 90 deletions(-)
 delete mode 100644 database/query/fetcher.go

diff --git a/database/controller.go b/database/controller.go
index 428f319..7a967d4 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -1,6 +1,7 @@
 package database
 
 import (
+	"errors"
 	"sync"
 	"time"
 
@@ -35,6 +36,9 @@ func (c *Controller) Get(key string) (record.Record, error) {
 		return nil, err
 	}
 
+	r.Lock()
+	defer r.Unlock()
+
 	if !r.Meta().CheckValidity(time.Now().Unix()) {
 		return nil, ErrNotFound
 	}
@@ -44,14 +48,27 @@ func (c *Controller) Get(key string) (record.Record, error) {
 
 // Put saves a record in the database.
 func (c *Controller) Put(r record.Record) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
 	return c.storage.Put(r)
 }
 
+// Delete a record from the database.
 func (c *Controller) Delete(key string) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
 	r, err := c.Get(key)
 	if err != nil {
 		return err
 	}
+
+	r.Lock()
+	defer r.Unlock()
+
 	r.Meta().Deleted = time.Now().Unix()
 	return c.Put(r)
 }
@@ -59,10 +76,36 @@ func (c *Controller) Delete(key string) error {
 // Partial
 // What happens if I mutate a value that does not yet exist? How would I know its type?
 func (c *Controller) InsertPartial(key string, partialObject interface{}) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
 	return nil
 }
 
 func (c *Controller) InsertValue(key string, attribute string, value interface{}) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
+	r, err := c.Get(key)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	if r.IsWrapped() {
+		wrapper, ok := r.(*record.Wrapper)
+		if !ok {
+			return errors.New("record is malformed")
+		}
+
+	} else {
+
+	}
+
 	return nil
 }
 
@@ -73,17 +116,33 @@ func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iter
 
 // Meta
 func (c *Controller) SetAbsoluteExpiry(key string, time int64) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
 	return nil
 }
 
 func (c *Controller) SetRelativateExpiry(key string, duration int64) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
 	return nil
 }
 
 func (c *Controller) MakeCrownJewel(key string) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
 	return nil
 }
 
 func (c *Controller) MakeSecret(key string) error {
+	if c.storage.ReadOnly() {
+		return ErrReadOnly
+	}
+
 	return nil
 }
diff --git a/database/database.go b/database/database.go
index 78373f4..890c391 100644
--- a/database/database.go
+++ b/database/database.go
@@ -8,8 +8,9 @@ import (
 
 // Errors
 var (
-	ErrNotFound         = errors.New("database: entry could not be found")
-	ErrPermissionDenied = errors.New("database: access to record denied")
+	ErrNotFound         = errors.New("database entry could not be found")
+	ErrPermissionDenied = errors.New("access to database record denied")
+	ErrReadOnly         = errors.New("database is read only")
 )
 
 func init() {
diff --git a/database/query/fetcher.go b/database/query/fetcher.go
deleted file mode 100644
index c2004bd..0000000
--- a/database/query/fetcher.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package query
-
-import (
-	"github.com/tidwall/gjson"
-)
-
-const (
-	emptyString = ""
-)
-
-// Fetcher provides an interface to supply the query matcher a method to retrieve values from an object.
-type Fetcher interface {
-	GetString(key string) (value string, ok bool)
-	GetInt(key string) (value int64, ok bool)
-	GetFloat(key string) (value float64, ok bool)
-	GetBool(key string) (value bool, ok bool)
-	Exists(key string) bool
-}
-
-// JSONFetcher is a json string with get functions.
-type JSONFetcher struct {
-	json string
-}
-
-// NewJSONFetcher adds the Fetcher interface to a JSON string.
-func NewJSONFetcher(json string) *JSONFetcher {
-	return &JSONFetcher{
-		json: json,
-	}
-}
-
-// GetString returns the string found by the given json key and whether it could be successfully extracted.
-func (jf *JSONFetcher) GetString(key string) (value string, ok bool) {
-	result := gjson.Get(jf.json, key)
-	if !result.Exists() || result.Type != gjson.String {
-		return emptyString, false
-	}
-	return result.String(), true
-}
-
-// GetInt returns the int found by the given json key and whether it could be successfully extracted.
-func (jf *JSONFetcher) GetInt(key string) (value int64, ok bool) {
-	result := gjson.Get(jf.json, key)
-	if !result.Exists() || result.Type != gjson.Number {
-		return 0, false
-	}
-	return result.Int(), true
-}
-
-// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
-func (jf *JSONFetcher) GetFloat(key string) (value float64, ok bool) {
-	result := gjson.Get(jf.json, key)
-	if !result.Exists() || result.Type != gjson.Number {
-		return 0, false
-	}
-	return result.Float(), true
-}
-
-// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
-func (jf *JSONFetcher) GetBool(key string) (value bool, ok bool) {
-	result := gjson.Get(jf.json, key)
-	switch {
-	case !result.Exists():
-		return false, false
-	case result.Type == gjson.True:
-		return true, true
-	case result.Type == gjson.False:
-		return false, true
-	default:
-		return false, false
-	}
-}
-
-// Exists returns the whether the given key exists.
-func (jf *JSONFetcher) Exists(key string) bool {
-	result := gjson.Get(jf.json, key)
-	return result.Exists()
-}
diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
index 21f75b9..af04bf4 100644
--- a/database/storage/badger/badger.go
+++ b/database/storage/badger/badger.go
@@ -2,13 +2,12 @@ package badger
 
 import (
 	"errors"
-	"time"
 
 	"github.com/dgraph-io/badger"
 
 	"github.com/Safing/portbase/database/iterator"
-	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/storage"
 )
 
@@ -85,7 +84,7 @@ func (b *Badger) Get(key string) (record.Record, error) {
 		return nil, err
 	}
 
-	m, err := model.NewRawWrapper(b.name, string(item.Key()), data)
+	m, err := record.NewRawWrapper(b.name, string(item.Key()), data)
 	if err != nil {
 		return nil, err
 	}
@@ -100,12 +99,7 @@ func (b *Badger) Put(m record.Record) error {
 	}
 
 	err = b.db.Update(func(txn *badger.Txn) error {
-		if m.Meta().GetAbsoluteExpiry() > 0 {
-			txn.SetWithTTL([]byte(m.DatabaseKey()), data, time.Duration(m.Meta().GetRelativeExpiry()))
-		} else {
-			txn.Set([]byte(m.DatabaseKey()), data)
-		}
-		return nil
+		return txn.Set([]byte(m.DatabaseKey()), data)
 	})
 	return err
 }
@@ -126,6 +120,11 @@ func (b *Badger) Query(q *query.Query) (*iterator.Iterator, error) {
 	return nil, errors.New("query not implemented by badger")
 }
 
+// ReadOnly returns whether the database is read only.
+func (b *Badger) ReadOnly() bool {
+	return false
+}
+
 // Maintain runs a light maintenance operation on the database.
 func (b *Badger) Maintain() error {
 	b.db.RunValueLogGC(0.7)
diff --git a/database/storage/interface.go b/database/storage/interface.go
index 3832ca4..a94b32f 100644
--- a/database/storage/interface.go
+++ b/database/storage/interface.go
@@ -13,6 +13,7 @@ type Interface interface {
 	Delete(key string) error
 	Query(q *query.Query) (*iterator.Iterator, error)
 
+	ReadOnly() bool
 	Maintain() error
 	MaintainThorough() error
 	Shutdown() error
diff --git a/database/storage/sinkhole/sinkhole.go b/database/storage/sinkhole/sinkhole.go
index 5a5189e..b1447a7 100644
--- a/database/storage/sinkhole/sinkhole.go
+++ b/database/storage/sinkhole/sinkhole.go
@@ -4,8 +4,8 @@ import (
 	"errors"
 
 	"github.com/Safing/portbase/database/iterator"
-	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/record"
 	"github.com/Safing/portbase/database/storage"
 )
 
@@ -50,6 +50,11 @@ func (s *Sinkhole) Query(q *query.Query) (*iterator.Iterator, error) {
 	return nil, errors.New("query not implemented by sinkhole")
 }
 
+// ReadOnly returns whether the database is read only.
+func (s *Sinkhole) ReadOnly() bool {
+	return false
+}
+
 // Maintain runs a light maintenance operation on the database.
 func (s *Sinkhole) Maintain() error {
 	return nil

From 06a34f931eb95576e8cbb1432a3ce828130375c6 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Mon, 10 Sep 2018 19:01:28 +0200
Subject: [PATCH 15/20] Finish minimal feature set, start with tests

---
 .../accessor-json-bytes.go                    |  22 ++-
 .../accessor-json-string.go                   |  22 ++-
 .../{record => accessor}/accessor-struct.go   |   2 +-
 database/{record => accessor}/accessor.go     |   2 +-
 .../{record => accessor}/accessor_test.go     |  92 +++++++--
 database/controller.go                        | 114 +++--------
 database/database.go                          | 121 ------------
 database/database_test.go                     |  69 +++++++
 database/databases.go                         |   7 +-
 database/dbmodule/db.go                       |  34 +++-
 database/dbmodule/maintenance.go              |  32 ++++
 database/errors.go                            |  15 ++
 database/interface.go                         | 179 ++++++++++++++++--
 database/location.go                          |  45 +++++
 database/maintainence.go                      |  50 +++++
 database/query/condition-and.go               |   6 +-
 database/query/condition-bool.go              |   6 +-
 database/query/condition-error.go             |   6 +-
 database/query/condition-exists.go            |   6 +-
 database/query/condition-float.go             |   6 +-
 database/query/condition-int.go               |   6 +-
 database/query/condition-no.go                |   6 +-
 database/query/condition-not.go               |   6 +-
 database/query/condition-or.go                |   6 +-
 database/query/condition-regex.go             |   6 +-
 database/query/condition-string.go            |   6 +-
 database/query/condition-stringslice.go       |   5 +-
 database/query/condition.go                   |   8 +-
 database/query/parser_test.go                 |   1 -
 database/query/query.go                       |  46 +++--
 database/query/query_test.go                  |   8 +-
 database/record/meta.go                       |   5 +
 database/registry.go                          |  50 +----
 database/storage/interface.go                 |   2 +-
 34 files changed, 651 insertions(+), 346 deletions(-)
 rename database/{record => accessor}/accessor-json-bytes.go (76%)
 rename database/{record => accessor}/accessor-json-string.go (76%)
 rename database/{record => accessor}/accessor-struct.go (99%)
 rename database/{record => accessor}/accessor.go (96%)
 rename database/{record => accessor}/accessor_test.go (67%)
 delete mode 100644 database/database.go
 create mode 100644 database/database_test.go
 create mode 100644 database/dbmodule/maintenance.go
 create mode 100644 database/errors.go
 create mode 100644 database/maintainence.go

diff --git a/database/record/accessor-json-bytes.go b/database/accessor/accessor-json-bytes.go
similarity index 76%
rename from database/record/accessor-json-bytes.go
rename to database/accessor/accessor-json-bytes.go
index ebb107f..08330d7 100644
--- a/database/record/accessor-json-bytes.go
+++ b/database/accessor/accessor-json-bytes.go
@@ -1,6 +1,8 @@
-package record
+package accessor
 
 import (
+	"fmt"
+
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
 )
@@ -19,6 +21,24 @@ func NewJSONBytesAccessor(json *[]byte) *JSONBytesAccessor {
 
 // Set sets the value identified by key.
 func (ja *JSONBytesAccessor) Set(key string, value interface{}) error {
+	result := gjson.GetBytes(*ja.json, key)
+	if result.Exists() {
+		switch value.(type) {
+		case string:
+			if result.Type != gjson.String {
+				return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
+			}
+		case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
+			if result.Type != gjson.Number {
+				return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
+			}
+		case bool:
+			if result.Type != gjson.True && result.Type != gjson.False {
+				return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
+			}
+		}
+	}
+
 	new, err := sjson.SetBytes(*ja.json, key, value)
 	if err != nil {
 		return err
diff --git a/database/record/accessor-json-string.go b/database/accessor/accessor-json-string.go
similarity index 76%
rename from database/record/accessor-json-string.go
rename to database/accessor/accessor-json-string.go
index 4a4bd61..1170418 100644
--- a/database/record/accessor-json-string.go
+++ b/database/accessor/accessor-json-string.go
@@ -1,6 +1,8 @@
-package record
+package accessor
 
 import (
+	"fmt"
+
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
 )
@@ -19,6 +21,24 @@ func NewJSONAccessor(json *string) *JSONAccessor {
 
 // Set sets the value identified by key.
 func (ja *JSONAccessor) Set(key string, value interface{}) error {
+	result := gjson.Get(*ja.json, key)
+	if result.Exists() {
+		switch value.(type) {
+		case string:
+			if result.Type != gjson.String {
+				return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
+			}
+		case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
+			if result.Type != gjson.Number {
+				return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
+			}
+		case bool:
+			if result.Type != gjson.True && result.Type != gjson.False {
+				return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
+			}
+		}
+	}
+
 	new, err := sjson.Set(*ja.json, key, value)
 	if err != nil {
 		return err
diff --git a/database/record/accessor-struct.go b/database/accessor/accessor-struct.go
similarity index 99%
rename from database/record/accessor-struct.go
rename to database/accessor/accessor-struct.go
index 9cafc39..b56c9b7 100644
--- a/database/record/accessor-struct.go
+++ b/database/accessor/accessor-struct.go
@@ -1,4 +1,4 @@
-package record
+package accessor
 
 import (
 	"errors"
diff --git a/database/record/accessor.go b/database/accessor/accessor.go
similarity index 96%
rename from database/record/accessor.go
rename to database/accessor/accessor.go
index d6856bc..aedad26 100644
--- a/database/record/accessor.go
+++ b/database/accessor/accessor.go
@@ -1,4 +1,4 @@
-package record
+package accessor
 
 const (
 	emptyString = ""
diff --git a/database/record/accessor_test.go b/database/accessor/accessor_test.go
similarity index 67%
rename from database/record/accessor_test.go
rename to database/accessor/accessor_test.go
index 12cbcac..d2a3a20 100644
--- a/database/record/accessor_test.go
+++ b/database/accessor/accessor_test.go
@@ -1,4 +1,4 @@
-package record
+package accessor
 
 import (
 	"encoding/json"
@@ -95,6 +95,16 @@ func testGetBool(t *testing.T, acc Accessor, key string, shouldSucceed bool, exp
 	}
 }
 
+func testExists(t *testing.T, acc Accessor, key string, shouldSucceed bool) {
+	ok := acc.Exists(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s should report key %s as existing", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should report key %s as non-existing", acc.Type(), key)
+	}
+}
+
 func testSet(t *testing.T, acc Accessor, key string, shouldSucceed bool, valueToSet interface{}) {
 	err := acc.Set(key, valueToSet)
 	switch {
@@ -150,7 +160,7 @@ func TestAccessor(t *testing.T) {
 		testSet(t, acc, "B", true, false)
 	}
 
-	// get again
+	// get again to check if new values were set
 	for _, acc := range accs {
 		testGetString(t, acc, "S", true, "coconut")
 		testGetInt(t, acc, "I", true, 44)
@@ -170,19 +180,69 @@ func TestAccessor(t *testing.T) {
 
 	// failures
 	for _, acc := range accs {
-		testGetString(t, acc, "S", false, 1)
-		testGetInt(t, acc, "I", false, 44)
-		testGetInt(t, acc, "I8", false, 512)
-		testGetInt(t, acc, "I16", false, 1000000)
-		testGetInt(t, acc, "I32", false, 44)
-		testGetInt(t, acc, "I64", false, "44")
-		testGetInt(t, acc, "UI", false, 44)
-		testGetInt(t, acc, "UI8", false, 44)
-		testGetInt(t, acc, "UI16", false, 44)
-		testGetInt(t, acc, "UI32", false, 44)
-		testGetInt(t, acc, "UI64", false, 44)
-		testGetFloat(t, acc, "F32", false, 44.44)
-		testGetFloat(t, acc, "F64", false, 44.44)
-		testGetBool(t, acc, "B", false, false)
+		testSet(t, acc, "S", false, true)
+		testSet(t, acc, "S", false, false)
+		testSet(t, acc, "S", false, 1)
+		testSet(t, acc, "S", false, 1.1)
+
+		testSet(t, acc, "I", false, "1")
+		testSet(t, acc, "I8", false, "1")
+		testSet(t, acc, "I16", false, "1")
+		testSet(t, acc, "I32", false, "1")
+		testSet(t, acc, "I64", false, "1")
+		testSet(t, acc, "UI", false, "1")
+		testSet(t, acc, "UI8", false, "1")
+		testSet(t, acc, "UI16", false, "1")
+		testSet(t, acc, "UI32", false, "1")
+		testSet(t, acc, "UI64", false, "1")
+
+		testSet(t, acc, "F32", false, "1.1")
+		testSet(t, acc, "F64", false, "1.1")
+
+		testSet(t, acc, "B", false, "false")
+		testSet(t, acc, "B", false, 1)
+		testSet(t, acc, "B", false, 1.1)
 	}
+
+	// get again to check if values werent changed when an error occurred
+	for _, acc := range accs {
+		testGetString(t, acc, "S", true, "coconut")
+		testGetInt(t, acc, "I", true, 44)
+		testGetInt(t, acc, "I8", true, 44)
+		testGetInt(t, acc, "I16", true, 44)
+		testGetInt(t, acc, "I32", true, 44)
+		testGetInt(t, acc, "I64", true, 44)
+		testGetInt(t, acc, "UI", true, 44)
+		testGetInt(t, acc, "UI8", true, 44)
+		testGetInt(t, acc, "UI16", true, 44)
+		testGetInt(t, acc, "UI32", true, 44)
+		testGetInt(t, acc, "UI64", true, 44)
+		testGetFloat(t, acc, "F32", true, 44.44)
+		testGetFloat(t, acc, "F64", true, 44.44)
+		testGetBool(t, acc, "B", true, false)
+	}
+
+	// test existence
+	for _, acc := range accs {
+		testExists(t, acc, "S", true)
+		testExists(t, acc, "I", true)
+		testExists(t, acc, "I8", true)
+		testExists(t, acc, "I16", true)
+		testExists(t, acc, "I32", true)
+		testExists(t, acc, "I64", true)
+		testExists(t, acc, "UI", true)
+		testExists(t, acc, "UI8", true)
+		testExists(t, acc, "UI16", true)
+		testExists(t, acc, "UI32", true)
+		testExists(t, acc, "UI64", true)
+		testExists(t, acc, "F32", true)
+		testExists(t, acc, "F64", true)
+		testExists(t, acc, "B", true)
+	}
+
+	// test non-existence
+	for _, acc := range accs {
+		testExists(t, acc, "X", false)
+	}
+
 }
diff --git a/database/controller.go b/database/controller.go
index 7a967d4..f8534dc 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -1,7 +1,6 @@
 package database
 
 import (
-	"errors"
 	"sync"
 	"time"
 
@@ -29,8 +28,17 @@ func newController(storageInt storage.Interface) (*Controller, error) {
 	}, nil
 }
 
+// ReadOnly returns whether the storage is read only.
+func (c *Controller) ReadOnly() bool {
+	return c.storage.ReadOnly()
+}
+
 // Get return the record with the given key.
 func (c *Controller) Get(key string) (record.Record, error) {
+	if shuttingDown.IsSet() {
+		return nil, ErrShuttingDown
+	}
+
 	r, err := c.storage.Get(key)
 	if err != nil {
 		return nil, err
@@ -48,101 +56,41 @@ func (c *Controller) Get(key string) (record.Record, error) {
 
 // Put saves a record in the database.
 func (c *Controller) Put(r record.Record) error {
+	if shuttingDown.IsSet() {
+		return ErrShuttingDown
+	}
+
 	if c.storage.ReadOnly() {
 		return ErrReadOnly
 	}
 
+	if r.Meta() == nil {
+		r.SetMeta(&record.Meta{})
+	}
+	r.Meta().Update()
+
 	return c.storage.Put(r)
 }
 
-// Delete a record from the database.
-func (c *Controller) Delete(key string) error {
-	if c.storage.ReadOnly() {
-		return ErrReadOnly
-	}
-
-	r, err := c.Get(key)
-	if err != nil {
-		return err
-	}
-
-	r.Lock()
-	defer r.Unlock()
-
-	r.Meta().Deleted = time.Now().Unix()
-	return c.Put(r)
-}
-
-// Partial
-// What happens if I mutate a value that does not yet exist? How would I know its type?
-func (c *Controller) InsertPartial(key string, partialObject interface{}) error {
-	if c.storage.ReadOnly() {
-		return ErrReadOnly
-	}
-
-	return nil
-}
-
-func (c *Controller) InsertValue(key string, attribute string, value interface{}) error {
-	if c.storage.ReadOnly() {
-		return ErrReadOnly
-	}
-
-	r, err := c.Get(key)
-	if err != nil {
-		return err
-	}
-
-	r.Lock()
-	defer r.Unlock()
-
-	if r.IsWrapped() {
-		wrapper, ok := r.(*record.Wrapper)
-		if !ok {
-			return errors.New("record is malformed")
-		}
-
-	} else {
-
-	}
-
-	return nil
-}
-
-// Query
+// Query executes the given query on the database.
 func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {
-	return nil, nil
-}
-
-// Meta
-func (c *Controller) SetAbsoluteExpiry(key string, time int64) error {
-	if c.storage.ReadOnly() {
-		return ErrReadOnly
+	if shuttingDown.IsSet() {
+		return nil, ErrShuttingDown
 	}
-
-	return nil
+	return c.storage.Query(q, local, internal)
 }
 
-func (c *Controller) SetRelativateExpiry(key string, duration int64) error {
-	if c.storage.ReadOnly() {
-		return ErrReadOnly
-	}
-
-	return nil
+// Maintain runs the Maintain method no the storage.
+func (c *Controller) Maintain() error {
+	return c.storage.Maintain()
 }
 
-func (c *Controller) MakeCrownJewel(key string) error {
-	if c.storage.ReadOnly() {
-		return ErrReadOnly
-	}
-
-	return nil
+// MaintainThorough runs the MaintainThorough method no the storage.
+func (c *Controller) MaintainThorough() error {
+	return c.storage.MaintainThorough()
 }
 
-func (c *Controller) MakeSecret(key string) error {
-	if c.storage.ReadOnly() {
-		return ErrReadOnly
-	}
-
-	return nil
+// Shutdown shuts down the storage.
+func (c *Controller) Shutdown() error {
+	return c.storage.Shutdown()
 }
diff --git a/database/database.go b/database/database.go
deleted file mode 100644
index 890c391..0000000
--- a/database/database.go
+++ /dev/null
@@ -1,121 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"errors"
-)
-
-// Errors
-var (
-	ErrNotFound         = errors.New("database entry could not be found")
-	ErrPermissionDenied = errors.New("access to database record denied")
-	ErrReadOnly         = errors.New("database is read only")
-)
-
-func init() {
-	// if strings.HasSuffix(os.Args[0], ".test") {
-	// 	// testing setup
-	// 	log.Warning("===== DATABASE RUNNING IN TEST MODE =====")
-	// 	db = channelshim.NewChanneledDatastore(ds.NewMapDatastore())
-	// 	return
-	// }
-
-	// sfsDB, err := simplefs.NewDatastore(meta.DatabaseDir())
-	// if err != nil {
-	// 	fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err)
-	// 	os.Exit(1)
-	// }
-
-	// ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{})
-	// if err != nil {
-	// 	fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err)
-	// 	os.Exit(1)
-	// }
-	//
-	// mapDB := ds.NewMapDatastore()
-	//
-	// db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{
-	// 	mount.Mount{
-	// 		Prefix:    ds.NewKey("/Run"),
-	// 		Datastore: mapDB,
-	// 	},
-	// 	mount.Mount{
-	// 		Prefix:    ds.NewKey("/"),
-	// 		Datastore: ldb,
-	// 	},
-	// }))
-
-}
-
-// func Batch() (ds.Batch, error) {
-//   return db.Batch()
-// }
-
-// func Close() error {
-//   return db.Close()
-// }
-
-// func Get(key *ds.Key) (Model, error) {
-// 	data, err := db.Get(*key)
-// 	if err != nil {
-// 		switch err {
-// 		case ds.ErrNotFound:
-// 			return nil, ErrNotFound
-// 		default:
-// 			return nil, err
-// 		}
-// 	}
-// 	model, ok := data.(Model)
-// 	if !ok {
-// 		return nil, errors.New("database did not return model")
-// 	}
-// 	return model, nil
-// }
-
-// func Has(key ds.Key) (exists bool, err error) {
-// 	return db.Has(key)
-// }
-//
-// func Create(key ds.Key, model Model) (err error) {
-// 	handleCreateSubscriptions(model)
-// 	err = db.Put(key, model)
-// 	if err != nil {
-// 		log.Tracef("database: failed to create entry %s: %s", key, err)
-// 	}
-// 	return err
-// }
-//
-// func Update(key ds.Key, model Model) (err error) {
-// 	handleUpdateSubscriptions(model)
-// 	err = db.Put(key, model)
-// 	if err != nil {
-// 		log.Tracef("database: failed to update entry %s: %s", key, err)
-// 	}
-// 	return err
-// }
-//
-// func Delete(key ds.Key) (err error) {
-// 	handleDeleteSubscriptions(&key)
-// 	return db.Delete(key)
-// }
-//
-// func Query(q dsq.Query) (dsq.Results, error) {
-// 	return db.Query(q)
-// }
-//
-// func RawGet(key ds.Key) (*dbutils.Wrapper, error) {
-// 	data, err := db.Get(key)
-// 	if err != nil {
-// 		return nil, err
-// 	}
-// 	wrapped, ok := data.(*dbutils.Wrapper)
-// 	if !ok {
-// 		return nil, errors.New("returned data is not a wrapper")
-// 	}
-// 	return wrapped, nil
-// }
-//
-// func RawPut(key ds.Key, value interface{}) error {
-// 	return db.Put(key, value)
-// }
diff --git a/database/database_test.go b/database/database_test.go
new file mode 100644
index 0000000..5f878a7
--- /dev/null
+++ b/database/database_test.go
@@ -0,0 +1,69 @@
+package database
+
+import (
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+
+	"github.com/Safing/portbase/database/record"
+)
+
+type TestRecord struct {
+	record.Base
+	lock sync.Mutex
+	S    string
+	I    int
+	I8   int8
+	I16  int16
+	I32  int32
+	I64  int64
+	UI   uint
+	UI8  uint8
+	UI16 uint16
+	UI32 uint32
+	UI64 uint64
+	F32  float32
+	F64  float64
+	B    bool
+}
+
+func (tr *TestRecord) Lock() {
+}
+
+func (tr *TestRecord) Unlock() {
+}
+
+func TestDatabase(t *testing.T) {
+
+	testDir, err := ioutil.TempDir("", "testing-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = Initialize(testDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(testDir) // clean up
+
+	err = RegisterDatabase(&RegisteredDatabase{
+		Name:        "testing",
+		Description: "Unit Test Database",
+		StorageType: "badger",
+		PrimaryAPI:  "",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db := NewInterface(nil)
+
+	new := &TestRecord{}
+	new.SetKey("testing:A")
+	err = db.Put(new)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
diff --git a/database/databases.go b/database/databases.go
index 6170c71..086e8be 100644
--- a/database/databases.go
+++ b/database/databases.go
@@ -6,6 +6,7 @@ import (
   "fmt"
   "path"
 
+	"github.com/tevino/abool"
   "github.com/Safing/portbase/database/storage"
 	"github.com/Safing/portbase/database/record"
 )
@@ -13,14 +14,16 @@ import (
 var (
 	databases     = make(map[string]*Controller)
 	databasesLock sync.Mutex
+
+	shuttingDown = abool.NewBool(false)
 )
 
-func splitKeyAndGetDatabase(key string) (dbKey string, db *Controller, err error) {
+func splitKeyAndGetDatabase(key string) (db *Controller, dbKey string, err error) {
   var dbName string
   dbName, dbKey = record.ParseKey(key)
   db, err = getDatabase(dbName)
   if err != nil {
-    return "", nil, err
+    return nil, "", err
   }
   return
 }
diff --git a/database/dbmodule/db.go b/database/dbmodule/db.go
index 6c38c1f..1b7d1d8 100644
--- a/database/dbmodule/db.go
+++ b/database/dbmodule/db.go
@@ -1,29 +1,43 @@
 package dbmodule
 
 import (
-  "github.com/Safing/portbase/database"
+	"errors"
+	"flag"
+	"sync"
+
+	"github.com/Safing/portbase/database"
+	"github.com/Safing/portbase/modules"
 )
 
 var (
-	databaseDir string
+	databaseDir    string
+	shutdownSignal = make(chan struct{})
+	maintenanceWg  sync.WaitGroup
 )
 
 func init() {
-  flag.StringVar(&databaseDir, "db", "", "set database directory")
+	flag.StringVar(&databaseDir, "db", "", "set database directory")
 
-  modules.Register("database", prep, start, stop)
+	modules.Register("database", prep, start, stop)
 }
 
 func prep() error {
-  if databaseDir == "" {
-    return errors.New("no database location specified, set with `-db=/path/to/db`")
-  }
+	if databaseDir == "" {
+		return errors.New("no database location specified, set with `-db=/path/to/db`")
+	}
+	return nil
 }
 
 func start() error {
-  return database.Initialize(databaseDir)
+	err := database.Initialize(databaseDir)
+	if err == nil {
+		go maintainer()
+	}
+	return err
 }
 
-func stop() {
-  return database.Shutdown()
+func stop() error {
+	close(shutdownSignal)
+	maintenanceWg.Wait()
+	return database.Shutdown()
 }
diff --git a/database/dbmodule/maintenance.go b/database/dbmodule/maintenance.go
new file mode 100644
index 0000000..df9a9eb
--- /dev/null
+++ b/database/dbmodule/maintenance.go
@@ -0,0 +1,32 @@
+package dbmodule
+
+import (
+  "time"
+
+  "github.com/Safing/portbase/database"
+  "github.com/Safing/portbase/log"
+)
+
+func maintainer() {
+  ticker := time.NewTicker(1 * time.Hour)
+  tickerThorough := time.NewTicker(10 * time.Minute)
+  maintenanceWg.Add(1)
+
+  for {
+    select {
+    case <- ticker.C:
+      err := database.Maintain()
+      if err != nil {
+        log.Errorf("database: maintenance error: %s", err)
+      }
+    case <- ticker.C:
+      err := database.MaintainThorough()
+      if err != nil {
+        log.Errorf("database: maintenance (thorough) error: %s", err)
+      }
+    case <-shutdownSignal:
+      maintenanceWg.Done()
+      return
+    }
+  }
+}
diff --git a/database/errors.go b/database/errors.go
new file mode 100644
index 0000000..55d42e6
--- /dev/null
+++ b/database/errors.go
@@ -0,0 +1,15 @@
+// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
+
+package database
+
+import (
+	"errors"
+)
+
+// Errors
+var (
+	ErrNotFound         = errors.New("database entry could not be found")
+	ErrPermissionDenied = errors.New("access to database record denied")
+	ErrReadOnly         = errors.New("database is read only")
+	ErrShuttingDown     = errors.New("database system is shutting down")
+)
diff --git a/database/interface.go b/database/interface.go
index c765559..b36c166 100644
--- a/database/interface.go
+++ b/database/interface.go
@@ -1,9 +1,19 @@
 package database
 
 import (
+	"errors"
+	"fmt"
+
+	"github.com/Safing/portbase/database/accessor"
+	"github.com/Safing/portbase/database/iterator"
+	"github.com/Safing/portbase/database/query"
 	"github.com/Safing/portbase/database/record"
 )
 
+const (
+	getDBFromKey = ""
+)
+
 // Interface provides a method to access the database with attached options.
 type Interface struct {
 	options *Options
@@ -30,7 +40,7 @@ func NewInterface(opts *Options) *Interface {
 
 // Exists return whether a record with the given key exists.
 func (i *Interface) Exists(key string) (bool, error) {
-	_, err := i.getRecord(key)
+	_, _, err := i.getRecord(getDBFromKey, key, false, false)
 	if err != nil {
 		if err == ErrNotFound {
 			return false, nil
@@ -42,28 +52,161 @@ func (i *Interface) Exists(key string) (bool, error) {
 
 // Get return the record with the given key.
 func (i *Interface) Get(key string) (record.Record, error) {
-	r, err := i.getRecord(key)
-	if err != nil {
-		return nil, err
-	}
-
-	if !r.Meta().CheckPermission(i.options.Local, i.options.Internal) {
-		return nil, ErrPermissionDenied
-	}
-
-	return r, nil
+	r, _, err := i.getRecord(getDBFromKey, key, true, false)
+	return r, err
 }
 
-func (i *Interface) getRecord(key string) (record.Record, error) {
-	dbKey, db, err := splitKeyAndGetDatabase(key)
-	if err != nil {
-		return nil, err
+func (i *Interface) getRecord(dbName string, dbKey string, check bool, mustBeWriteable bool) (r record.Record, db *Controller, err error) {
+	if dbName == "" {
+		dbName, dbKey = record.ParseKey(dbKey)
 	}
 
-	r, err := db.Get(dbKey)
+	db, err = getDatabase(dbName)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	return r, nil
+	if mustBeWriteable && db.ReadOnly() {
+		return nil, nil, ErrReadOnly
+	}
+
+	r, err = db.Get(dbKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if check && !r.Meta().CheckPermission(i.options.Local, i.options.Internal) {
+		return nil, nil, ErrPermissionDenied
+	}
+
+	return r, db, nil
+}
+
+// InsertValue inserts a value into a record.
+func (i *Interface) InsertValue(key string, attribute string, value interface{}) error {
+	r, db, err := i.getRecord(getDBFromKey, key, true, true)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	var acc accessor.Accessor
+	if r.IsWrapped() {
+		wrapper, ok := r.(*record.Wrapper)
+		if !ok {
+			return errors.New("record is malformed (reports to be wrapped but is not of type *record.Wrapper)")
+		}
+		acc = accessor.NewJSONBytesAccessor(&wrapper.Data)
+	} else {
+		acc = accessor.NewStructAccessor(r)
+	}
+
+	err = acc.Set(attribute, value)
+	if err != nil {
+		return fmt.Errorf("failed to set value with %s: %s", acc.Type(), err)
+	}
+
+	return db.Put(r)
+}
+
+// Put saves a record to the database.
+func (i *Interface) Put(r record.Record) error {
+	_, db, err := i.getRecord(r.DatabaseName(), r.DatabaseKey(), true, true)
+	if err != nil {
+		return err
+	}
+	return db.Put(r)
+}
+
+// PutNew saves a record to the database as a new record (ie. with a new creation timestamp).
+func (i *Interface) PutNew(r record.Record) error {
+	_, db, err := i.getRecord(r.DatabaseName(), r.DatabaseKey(), true, true)
+	if err != nil && err != ErrNotFound {
+		return err
+	}
+
+	r.SetMeta(&record.Meta{})
+	return db.Put(r)
+}
+
+// SetAbsoluteExpiry sets an absolute record expiry.
+func (i *Interface) SetAbsoluteExpiry(key string, time int64) error {
+	r, db, err := i.getRecord(getDBFromKey, key, true, true)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	r.Meta().SetAbsoluteExpiry(time)
+	return db.Put(r)
+}
+
+// SetRelativateExpiry sets a relative (self-updating) record expiry.
+func (i *Interface) SetRelativateExpiry(key string, duration int64) error {
+	r, db, err := i.getRecord(getDBFromKey, key, true, true)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	r.Meta().SetRelativateExpiry(duration)
+	return db.Put(r)
+}
+
+// MakeSecret marks the record as a secret, meaning interfacing processes, such as an UI, are denied access to the record.
+func (i *Interface) MakeSecret(key string) error {
+	r, db, err := i.getRecord(getDBFromKey, key, true, true)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	r.Meta().MakeSecret()
+	return db.Put(r)
+}
+
+// MakeCrownJewel marks a record as a crown jewel, meaning it will only be accessible locally.
+func (i *Interface) MakeCrownJewel(key string) error {
+	r, db, err := i.getRecord(getDBFromKey, key, true, true)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	r.Meta().MakeCrownJewel()
+	return db.Put(r)
+}
+
+// Delete deletes a record from the database.
+func (i *Interface) Delete(key string) error {
+	r, db, err := i.getRecord(getDBFromKey, key, true, true)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	r.Meta().Delete()
+	return db.Put(r)
+}
+
+// Query executes the given query on the database.
+func (i *Interface) Query(q *query.Query) (*iterator.Iterator, error) {
+	db, err := getDatabase(q.DatabaseName())
+	if err != nil {
+		return nil, err
+	}
+
+	return db.Query(q, i.options.Local, i.options.Internal)
 }
diff --git a/database/location.go b/database/location.go
index c0fdb78..a44e415 100644
--- a/database/location.go
+++ b/database/location.go
@@ -2,12 +2,57 @@ package database
 
 import (
 	"path"
+	"os"
+	"fmt"
+	"errors"
 )
 
 var (
 	rootDir string
 )
 
+// Initialize initialized the database
+func Initialize(location string) error {
+	if initialized.SetToIf(false, true) {
+		rootDir = location
+
+		err := checkRootDir()
+		if err != nil {
+			return fmt.Errorf("could not create/open database directory (%s): %s", rootDir, err)
+		}
+
+		err = loadRegistry()
+		if err != nil {
+			return fmt.Errorf("could not load database registry (%s): %s", path.Join(rootDir, registryFileName), err)
+		}
+
+		return nil
+	}
+	return errors.New("database already initialized")
+}
+
+func checkRootDir() error {
+	// open dir
+	dir, err := os.Open(rootDir)
+	if err != nil {
+		if err == os.ErrNotExist {
+			return os.MkdirAll(rootDir, 0700)
+		}
+		return err
+	}
+	defer dir.Close()
+
+	fileInfo, err := dir.Stat()
+	if err != nil {
+		return err
+	}
+
+	if fileInfo.Mode().Perm() != 0700 {
+		return dir.Chmod(0700)
+	}
+	return nil
+}
+
 // getLocation returns the storage location for the given name and type.
 func getLocation(name, storageType string) (location string, err error) {
 	return path.Join(rootDir, name, storageType), nil
diff --git a/database/maintainence.go b/database/maintainence.go
new file mode 100644
index 0000000..5b5f1f5
--- /dev/null
+++ b/database/maintainence.go
@@ -0,0 +1,50 @@
+package database
+
+// Maintain runs the Maintain method on all storages.
+func Maintain() (err error) {
+  controllers := duplicateControllers()
+  for _, c := range controllers {
+    err = c.Maintain()
+    if err != nil {
+      return
+    }
+  }
+  return
+}
+
+// MaintainThorough runs the MaintainThorough method on all storages.
+func MaintainThorough() (err error) {
+  controllers := duplicateControllers()
+  for _, c := range controllers {
+    err = c.MaintainThorough()
+    if err != nil {
+      return
+    }
+  }
+  return
+}
+
+// Shutdown shuts down the whole database system.
+func Shutdown() (err error) {
+  shuttingDown.Set()
+
+  controllers := duplicateControllers()
+  for _, c := range controllers {
+    err = c.Shutdown()
+    if err != nil {
+      return
+    }
+  }
+  return
+}
+
+func duplicateControllers() (controllers []*Controller) {
+  databasesLock.Lock()
+  defer databasesLock.Unlock()
+
+  for _, c := range databases {
+    controllers = append(controllers, c)
+  }
+
+  return
+}
diff --git a/database/query/condition-and.go b/database/query/condition-and.go
index 2346b78..74304b9 100644
--- a/database/query/condition-and.go
+++ b/database/query/condition-and.go
@@ -3,6 +3,8 @@ package query
 import (
 	"fmt"
 	"strings"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 // And combines multiple conditions with a logical _AND_ operator.
@@ -16,9 +18,9 @@ type andCond struct {
 	conditions []Condition
 }
 
-func (c *andCond) complies(f Fetcher) bool {
+func (c *andCond) complies(acc accessor.Accessor) bool {
 	for _, cond := range c.conditions {
-		if !cond.complies(f) {
+		if !cond.complies(acc) {
 			return false
 		}
 	}
diff --git a/database/query/condition-bool.go b/database/query/condition-bool.go
index e9d7c2b..834b592 100644
--- a/database/query/condition-bool.go
+++ b/database/query/condition-bool.go
@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 type boolCondition struct {
@@ -42,8 +44,8 @@ func newBoolCondition(key string, operator uint8, value interface{}) *boolCondit
 	}
 }
 
-func (c *boolCondition) complies(f Fetcher) bool {
-	comp, ok := f.GetBool(c.key)
+func (c *boolCondition) complies(acc accessor.Accessor) bool {
+	comp, ok := acc.GetBool(c.key)
 	if !ok {
 		return false
 	}
diff --git a/database/query/condition-error.go b/database/query/condition-error.go
index ab5b405..a46c36b 100644
--- a/database/query/condition-error.go
+++ b/database/query/condition-error.go
@@ -1,5 +1,9 @@
 package query
 
+import (
+	"github.com/Safing/portbase/database/accessor"
+)
+
 type errorCondition struct {
 	err error
 }
@@ -10,7 +14,7 @@ func newErrorCondition(err error) *errorCondition {
 	}
 }
 
-func (c *errorCondition) complies(f Fetcher) bool {
+func (c *errorCondition) complies(acc accessor.Accessor) bool {
 	return false
 }
 
diff --git a/database/query/condition-exists.go b/database/query/condition-exists.go
index 2e2b013..567360f 100644
--- a/database/query/condition-exists.go
+++ b/database/query/condition-exists.go
@@ -3,6 +3,8 @@ package query
 import (
 	"errors"
 	"fmt"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 type existsCondition struct {
@@ -17,8 +19,8 @@ func newExistsCondition(key string, operator uint8) *existsCondition {
 	}
 }
 
-func (c *existsCondition) complies(f Fetcher) bool {
-	return f.Exists(c.key)
+func (c *existsCondition) complies(acc accessor.Accessor) bool {
+	return acc.Exists(c.key)
 }
 
 func (c *existsCondition) check() error {
diff --git a/database/query/condition-float.go b/database/query/condition-float.go
index c34cf7f..4416594 100644
--- a/database/query/condition-float.go
+++ b/database/query/condition-float.go
@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 type floatCondition struct {
@@ -62,8 +64,8 @@ func newFloatCondition(key string, operator uint8, value interface{}) *floatCond
 	}
 }
 
-func (c *floatCondition) complies(f Fetcher) bool {
-	comp, ok := f.GetFloat(c.key)
+func (c *floatCondition) complies(acc accessor.Accessor) bool {
+	comp, ok := acc.GetFloat(c.key)
 	if !ok {
 		return false
 	}
diff --git a/database/query/condition-int.go b/database/query/condition-int.go
index 8f18ab2..dccac28 100644
--- a/database/query/condition-int.go
+++ b/database/query/condition-int.go
@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 type intCondition struct {
@@ -58,8 +60,8 @@ func newIntCondition(key string, operator uint8, value interface{}) *intConditio
 	}
 }
 
-func (c *intCondition) complies(f Fetcher) bool {
-	comp, ok := f.GetInt(c.key)
+func (c *intCondition) complies(acc accessor.Accessor) bool {
+	comp, ok := acc.GetInt(c.key)
 	if !ok {
 		return false
 	}
diff --git a/database/query/condition-no.go b/database/query/condition-no.go
index ff72fcc..8709474 100644
--- a/database/query/condition-no.go
+++ b/database/query/condition-no.go
@@ -1,9 +1,13 @@
 package query
 
+import (
+	"github.com/Safing/portbase/database/accessor"
+)
+
 type noCond struct {
 }
 
-func (c *noCond) complies(f Fetcher) bool {
+func (c *noCond) complies(acc accessor.Accessor) bool {
 	return true
 }
 
diff --git a/database/query/condition-not.go b/database/query/condition-not.go
index d395bc9..cac04a7 100644
--- a/database/query/condition-not.go
+++ b/database/query/condition-not.go
@@ -3,6 +3,8 @@ package query
 import (
 	"fmt"
 	"strings"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 // Not negates the supplied condition.
@@ -16,8 +18,8 @@ type notCond struct {
 	notC Condition
 }
 
-func (c *notCond) complies(f Fetcher) bool {
-	return !c.notC.complies(f)
+func (c *notCond) complies(acc accessor.Accessor) bool {
+	return !c.notC.complies(acc)
 }
 
 func (c *notCond) check() error {
diff --git a/database/query/condition-or.go b/database/query/condition-or.go
index d790f48..25fd37b 100644
--- a/database/query/condition-or.go
+++ b/database/query/condition-or.go
@@ -3,6 +3,8 @@ package query
 import (
 	"fmt"
 	"strings"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 // Or combines multiple conditions with a logical _OR_ operator.
@@ -16,9 +18,9 @@ type orCond struct {
 	conditions []Condition
 }
 
-func (c *orCond) complies(f Fetcher) bool {
+func (c *orCond) complies(acc accessor.Accessor) bool {
 	for _, cond := range c.conditions {
-		if cond.complies(f) {
+		if cond.complies(acc) {
 			return true
 		}
 	}
diff --git a/database/query/condition-regex.go b/database/query/condition-regex.go
index d795c84..e808fcd 100644
--- a/database/query/condition-regex.go
+++ b/database/query/condition-regex.go
@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 type regexCondition struct {
@@ -35,8 +37,8 @@ func newRegexCondition(key string, operator uint8, value interface{}) *regexCond
 	}
 }
 
-func (c *regexCondition) complies(f Fetcher) bool {
-	comp, ok := f.GetString(c.key)
+func (c *regexCondition) complies(acc accessor.Accessor) bool {
+	comp, ok := acc.GetString(c.key)
 	if !ok {
 		return false
 	}
diff --git a/database/query/condition-string.go b/database/query/condition-string.go
index 747e337..ddbf1b1 100644
--- a/database/query/condition-string.go
+++ b/database/query/condition-string.go
@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 type stringCondition struct {
@@ -28,8 +30,8 @@ func newStringCondition(key string, operator uint8, value interface{}) *stringCo
 	}
 }
 
-func (c *stringCondition) complies(f Fetcher) bool {
-	comp, ok := f.GetString(c.key)
+func (c *stringCondition) complies(acc accessor.Accessor) bool {
+	comp, ok := acc.GetString(c.key)
 	if !ok {
 		return false
 	}
diff --git a/database/query/condition-stringslice.go b/database/query/condition-stringslice.go
index ab3004d..ffc6643 100644
--- a/database/query/condition-stringslice.go
+++ b/database/query/condition-stringslice.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/Safing/portbase/database/accessor"
 	"github.com/Safing/portbase/utils"
 )
 
@@ -44,8 +45,8 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
 
 }
 
-func (c *stringSliceCondition) complies(f Fetcher) bool {
-	comp, ok := f.GetString(c.key)
+func (c *stringSliceCondition) complies(acc accessor.Accessor) bool {
+	comp, ok := acc.GetString(c.key)
 	if !ok {
 		return false
 	}
diff --git a/database/query/condition.go b/database/query/condition.go
index 3c5e7d2..52f4e89 100644
--- a/database/query/condition.go
+++ b/database/query/condition.go
@@ -1,10 +1,14 @@
 package query
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/Safing/portbase/database/accessor"
+)
 
 // Condition is an interface to provide a common api to all condition types.
 type Condition interface {
-	complies(f Fetcher) bool
+	complies(acc accessor.Accessor) bool
 	check() error
 	string() string
 }
diff --git a/database/query/parser_test.go b/database/query/parser_test.go
index e467100..7bd2bda 100644
--- a/database/query/parser_test.go
+++ b/database/query/parser_test.go
@@ -144,7 +144,6 @@ func testParseError(t *testing.T, queryText string, expectedErrorString string)
 func TestParseErrors(t *testing.T) {
 	// syntax
 	testParseError(t, `query`, `unexpected end at position 5`)
-	testParseError(t, `query test`, `invalid prefix: test`)
 	testParseError(t, `query test: where`, `unexpected end at position 17`)
 	testParseError(t, `query test: where (`, `unexpected end at position 19`)
 	testParseError(t, `query test: where )`, `unknown clause ")" at position 19`)
diff --git a/database/query/query.go b/database/query/query.go
index 8b4433e..0f23023 100644
--- a/database/query/query.go
+++ b/database/query/query.go
@@ -2,8 +2,10 @@ package query
 
 import (
 	"fmt"
-	"regexp"
 	"strings"
+
+	"github.com/Safing/portbase/database/accessor"
+	"github.com/Safing/portbase/database/record"
 )
 
 // Example:
@@ -16,24 +18,23 @@ import (
 //     )
 //   )
 
-var (
-	prefixExpr = regexp.MustCompile("^[a-z-]+:")
-)
-
 // Query contains a compiled query.
 type Query struct {
-	checked bool
-	prefix  string
-	where   Condition
-	orderBy string
-	limit   int
-	offset  int
+	checked     bool
+	dbName      string
+	dbKeyPrefix string
+	where       Condition
+	orderBy     string
+	limit       int
+	offset      int
 }
 
 // New creates a new query with the supplied prefix.
 func New(prefix string) *Query {
+	dbName, dbKeyPrefix := record.ParseKey(prefix)
 	return &Query{
-		prefix: prefix,
+		dbName:      dbName,
+		dbKeyPrefix: dbKeyPrefix,
 	}
 }
 
@@ -67,11 +68,6 @@ func (q *Query) Check() (*Query, error) {
 		return q, nil
 	}
 
-	// check prefix
-	if !prefixExpr.MatchString(q.prefix) {
-		return nil, fmt.Errorf("invalid prefix: %s", q.prefix)
-	}
-
 	// check condition
 	if q.where != nil {
 		err := q.where.check()
@@ -101,8 +97,8 @@ func (q *Query) IsChecked() bool {
 }
 
 // Matches checks whether the query matches the supplied data object.
-func (q *Query) Matches(f Fetcher) bool {
-	return q.where.complies(f)
+func (q *Query) Matches(acc accessor.Accessor) bool {
+	return q.where.complies(acc)
 }
 
 // Print returns the string representation of the query.
@@ -130,5 +126,15 @@ func (q *Query) Print() string {
 		offset = fmt.Sprintf(" offset %d", q.offset)
 	}
 
-	return fmt.Sprintf("query %s%s%s%s%s", q.prefix, where, orderBy, limit, offset)
+	return fmt.Sprintf("query %s:%s%s%s%s%s", q.dbName, q.dbKeyPrefix, where, orderBy, limit, offset)
+}
+
+// DatabaseName returns the name of the database.
+func (q *Query) DatabaseName() string {
+	return q.dbName
+}
+
+// DatabaseKeyPrefix returns the key prefix for the database.
+func (q *Query) DatabaseKeyPrefix() string {
+	return q.dbKeyPrefix
 }
diff --git a/database/query/query_test.go b/database/query/query_test.go
index bf25e70..6ac49f8 100644
--- a/database/query/query_test.go
+++ b/database/query/query_test.go
@@ -2,6 +2,8 @@ package query
 
 import (
 	"testing"
+
+	"github.com/Safing/portbase/database/accessor"
 )
 
 var (
@@ -44,12 +46,12 @@ var (
 }`
 )
 
-func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) {
+func testQuery(t *testing.T, acc accessor.Accessor, shouldMatch bool, condition Condition) {
 	q := New("test:").Where(condition).MustBeValid()
 
 	// fmt.Printf("%s\n", q.String())
 
-	matched := q.Matches(f)
+	matched := q.Matches(acc)
 	switch {
 	case !matched && shouldMatch:
 		t.Errorf("should match: %s", q.Print())
@@ -63,7 +65,7 @@ func TestQuery(t *testing.T) {
 	// if !gjson.Valid(testJSON) {
 	// 	t.Fatal("test json is invalid")
 	// }
-	f := NewJSONFetcher(testJSON)
+	f := accessor.NewJSONAccessor(&testJSON)
 
 	testQuery(t, f, true, Where("age", Equals, 100))
 	testQuery(t, f, true, Where("age", GreaterThan, uint8(99)))
diff --git a/database/record/meta.go b/database/record/meta.go
index 8ef2810..d916816 100644
--- a/database/record/meta.go
+++ b/database/record/meta.go
@@ -73,6 +73,11 @@ func (m *Meta) Reset() {
 	m.Deleted = 0
 }
 
+// Delete marks the record as deleted.
+func (m *Meta) Delete() {
+	m.Deleted = time.Now().Unix()
+}
+
 // CheckValidity checks whether the database record is valid.
 func (m *Meta) CheckValidity(now int64) (valid bool) {
 	switch {
diff --git a/database/registry.go b/database/registry.go
index 94734af..c83a9af 100644
--- a/database/registry.go
+++ b/database/registry.go
@@ -3,10 +3,10 @@ package database
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
+	"regexp"
 	"sync"
 
 	"github.com/tevino/abool"
@@ -40,6 +40,8 @@ var (
 
 	registry     map[string]*RegisteredDatabase
 	registryLock sync.Mutex
+
+	nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{5,}$")
 )
 
 // RegisterDatabase registers a new database.
@@ -48,6 +50,10 @@ func RegisterDatabase(new *RegisteredDatabase) error {
 		return errors.New("database not initialized")
 	}
 
+	if !nameConstraint.MatchString(new.Name) {
+		return errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 5 characters long")
+	}
+
 	registryLock.Lock()
 	defer registryLock.Unlock()
 
@@ -60,48 +66,6 @@ func RegisterDatabase(new *RegisteredDatabase) error {
 	return nil
 }
 
-// Initialize initialized the database
-func Initialize(location string) error {
-	if initialized.SetToIf(false, true) {
-		rootDir = location
-
-		err := checkRootDir()
-		if err != nil {
-			return fmt.Errorf("could not create/open database directory (%s): %s", rootDir, err)
-		}
-
-		err = loadRegistry()
-		if err != nil {
-			return fmt.Errorf("could not load database registry (%s): %s", path.Join(rootDir, registryFileName), err)
-		}
-
-		return nil
-	}
-	return errors.New("database already initialized")
-}
-
-func checkRootDir() error {
-	// open dir
-	dir, err := os.Open(rootDir)
-	if err != nil {
-		if err == os.ErrNotExist {
-			return os.MkdirAll(rootDir, 0700)
-		}
-		return err
-	}
-	defer dir.Close()
-
-	fileInfo, err := dir.Stat()
-	if err != nil {
-		return err
-	}
-
-	if fileInfo.Mode().Perm() != 0700 {
-		return dir.Chmod(0700)
-	}
-	return nil
-}
-
 func loadRegistry() error {
 	registryLock.Lock()
 	defer registryLock.Unlock()
diff --git a/database/storage/interface.go b/database/storage/interface.go
index a94b32f..b5bd803 100644
--- a/database/storage/interface.go
+++ b/database/storage/interface.go
@@ -11,7 +11,7 @@ type Interface interface {
 	Get(key string) (record.Record, error)
 	Put(m record.Record) error
 	Delete(key string) error
-	Query(q *query.Query) (*iterator.Iterator, error)
+	Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
 
 	ReadOnly() bool
 	Maintain() error

From 4802982734ccf867869bb36c982e10ed7c7e8b83 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 11 Sep 2018 18:59:27 +0200
Subject: [PATCH 16/20] Work on database + tests

---
 database/controller.go                    |  10 +-
 database/{databases.go => controllers.go} |  65 +++++------
 database/database.go                      |  32 ++++++
 database/database_test.go                 |  63 +++++++++--
 database/interface.go                     |  35 +++++-
 database/location.go                      |  53 ++++-----
 database/main.go                          |  55 ++++++++++
 database/maintainence.go                  |  26 +----
 database/registry.go                      | 127 +++++++++++++++-------
 database/storage/badger/badger.go         |   2 +-
 database/storage/badger/badger_test.go    |  69 ++++++++++++
 database/storage/errors.go                |   2 +-
 database/storage/storages.go              |   6 +-
 13 files changed, 395 insertions(+), 150 deletions(-)
 rename database/{databases.go => controllers.go} (53%)
 create mode 100644 database/database.go
 create mode 100644 database/main.go
 create mode 100644 database/storage/badger/badger_test.go

diff --git a/database/controller.go b/database/controller.go
index f8534dc..2c4875f 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -17,7 +17,8 @@ type Controller struct {
 	storage   storage.Interface
 	writeLock sync.RWMutex
 	readLock  sync.RWMutex
-	migrating *abool.AtomicBool
+	migrating *abool.AtomicBool // TODO
+	hibernating *abool.AtomicBool // TODO
 }
 
 // newController creates a new controller for a storage.
@@ -25,6 +26,7 @@ func newController(storageInt storage.Interface) (*Controller, error) {
 	return &Controller{
 		storage:   storageInt,
 		migrating: abool.NewBool(false),
+		hibernating: abool.NewBool(false),
 	}, nil
 }
 
@@ -41,6 +43,10 @@ func (c *Controller) Get(key string) (record.Record, error) {
 
 	r, err := c.storage.Get(key)
 	if err != nil {
+		// replace not found error
+		if err == storage.ErrNotFound {
+			return nil, ErrNotFound
+		}
 		return nil, err
 	}
 
@@ -60,7 +66,7 @@ func (c *Controller) Put(r record.Record) error {
 		return ErrShuttingDown
 	}
 
-	if c.storage.ReadOnly() {
+	if c.ReadOnly() {
 		return ErrReadOnly
 	}
 
diff --git a/database/databases.go b/database/controllers.go
similarity index 53%
rename from database/databases.go
rename to database/controllers.go
index 086e8be..5b9d002 100644
--- a/database/databases.go
+++ b/database/controllers.go
@@ -4,74 +4,63 @@ import (
 	"errors"
 	"sync"
   "fmt"
-  "path"
 
-	"github.com/tevino/abool"
   "github.com/Safing/portbase/database/storage"
-	"github.com/Safing/portbase/database/record"
 )
 
 var (
-	databases     = make(map[string]*Controller)
-	databasesLock sync.Mutex
-
-	shuttingDown = abool.NewBool(false)
+	controllers     = make(map[string]*Controller)
+	controllersLock sync.Mutex
 )
 
-func splitKeyAndGetDatabase(key string) (db *Controller, dbKey string, err error) {
-  var dbName string
-  dbName, dbKey = record.ParseKey(key)
-  db, err = getDatabase(dbName)
-  if err != nil {
-    return nil, "", err
-  }
-  return
-}
-
-func getDatabase(name string) (*Controller, error) {
+func getController(name string) (*Controller, error) {
   if !initialized.IsSet() {
     return nil, errors.New("database not initialized")
   }
 
-	databasesLock.Lock()
-	defer databasesLock.Unlock()
+	controllersLock.Lock()
+	defer controllersLock.Unlock()
 
   // return database if already started
-	db, ok := databases[name]
+	controller, ok := controllers[name]
 	if ok {
-    return db, nil
+    return controller, nil
 	}
 
-  registryLock.Lock()
-  defer registryLock.Unlock()
+	// get db registration
+	registeredDB, err := getDatabase(name)
+	if err != nil {
+		return nil, fmt.Errorf(`could not start database %s (type %s): %s`, name, registeredDB.StorageType, err)
+	}
 
-  // check if database exists at all
-  registeredDB, ok := registry[name]
-  if !ok {
-    return nil, fmt.Errorf(`database "%s" not registered`, name)
-  }
+	// get location
+	dbLocation, err := getLocation(name, registeredDB.StorageType)
+	if err != nil {
+		return nil, fmt.Errorf(`could not start database %s (type %s): %s`, name, registeredDB.StorageType, err)
+	}
 
   // start database
-  storageInt, err := storage.StartDatabase(name, registeredDB.StorageType, path.Join(rootDir, name, registeredDB.StorageType))
+  storageInt, err := storage.StartDatabase(name, registeredDB.StorageType, dbLocation)
   if err != nil {
     return nil, fmt.Errorf(`could not start database %s (type %s): %s`, name, registeredDB.StorageType, err)
   }
 
-  db, err = newController(storageInt)
+	// create controller
+  controller, err = newController(storageInt)
   if err != nil {
     return nil, fmt.Errorf(`could not create controller for database %s: %s`, name, err)
   }
 
-  databases[name] = db
-  return db, nil
+  controllers[name] = controller
+  return controller, nil
 }
 
 // InjectDatabase injects an already running database into the system.
 func InjectDatabase(name string, storageInt storage.Interface) error {
-	databasesLock.Lock()
-	defer databasesLock.Unlock()
+	controllersLock.Lock()
+	defer controllersLock.Unlock()
 
-	_, ok := databases[name]
+	_, ok := controllers[name]
 	if ok {
 		return errors.New(`database "%s" already loaded`)
 	}
@@ -88,11 +77,11 @@ func InjectDatabase(name string, storageInt storage.Interface) error {
     return fmt.Errorf(`database not of type "injected"`)
   }
 
-  db, err := newController(storageInt)
+  controller, err := newController(storageInt)
   if err != nil {
     return fmt.Errorf(`could not create controller for database %s: %s`, name, err)
   }
 
-	databases[name] = db
+	controllers[name] = controller
 	return nil
 }
diff --git a/database/database.go b/database/database.go
new file mode 100644
index 0000000..e8e4504
--- /dev/null
+++ b/database/database.go
@@ -0,0 +1,32 @@
+package database
+
+import (
+	"errors"
+	"time"
+)
+
+// Database holds information about registered databases
+type Database struct {
+	Name        string
+	Description string
+	StorageType string
+	PrimaryAPI  string
+	Registered  time.Time
+	LastUpdated time.Time
+	LastLoaded  time.Time
+}
+
+// MigrateTo migrates the database to another storage type.
+func (db *Database) MigrateTo(newStorageType string) error {
+	return errors.New("not implemented yet") // TODO
+}
+
+// Loaded updates the LastLoaded timestamp.
+func (db *Database) Loaded() {
+	db.LastLoaded = time.Now().Round(time.Second)
+}
+
+// Updated updates the LastUpdated timestamp.
+func (db *Database) Updated() {
+	db.LastUpdated = time.Now().Round(time.Second)
+}
diff --git a/database/database_test.go b/database/database_test.go
index 5f878a7..bab7579 100644
--- a/database/database_test.go
+++ b/database/database_test.go
@@ -1,12 +1,16 @@
 package database
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
+	"runtime/pprof"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/Safing/portbase/database/record"
+	_ "github.com/Safing/portbase/database/storage/badger"
 )
 
 type TestRecord struct {
@@ -34,7 +38,46 @@ func (tr *TestRecord) Lock() {
 func (tr *TestRecord) Unlock() {
 }
 
-func TestDatabase(t *testing.T) {
+func makeKey(storageType, key string) string {
+	return fmt.Sprintf("%s:%s", storageType, key)
+}
+
+func testDatabase(t *testing.T, storageType string) {
+	dbName := fmt.Sprintf("testing-%s", storageType)
+	_, err := Register(&Database{
+		Name:        dbName,
+		Description: fmt.Sprintf("Unit Test Database for %s", storageType),
+		StorageType: storageType,
+		PrimaryAPI:  "",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db := NewInterface(nil)
+
+	new := &TestRecord{}
+	new.SetKey(makeKey(dbName, "A"))
+	err = db.Put(new)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.Get(makeKey(dbName, "A"))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestDatabaseSystem(t *testing.T) {
+
+	// panic after 10 seconds, to check for locks
+	go func() {
+		time.Sleep(10 * time.Second)
+		fmt.Println("===== TAKING TOO LONG FOR SHUTDOWN - PRINTING STACK TRACES =====")
+		pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+		os.Exit(1)
+	}()
 
 	testDir, err := ioutil.TempDir("", "testing-")
 	if err != nil {
@@ -47,21 +90,19 @@ func TestDatabase(t *testing.T) {
 	}
 	defer os.RemoveAll(testDir) // clean up
 
-	err = RegisterDatabase(&RegisteredDatabase{
-		Name:        "testing",
-		Description: "Unit Test Database",
-		StorageType: "badger",
-		PrimaryAPI:  "",
-	})
+	testDatabase(t, "badger")
+
+	err = Maintain()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	db := NewInterface(nil)
+	err = MaintainThorough()
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	new := &TestRecord{}
-	new.SetKey("testing:A")
-	err = db.Put(new)
+	err = Shutdown()
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/database/interface.go b/database/interface.go
index b36c166..de5910c 100644
--- a/database/interface.go
+++ b/database/interface.go
@@ -27,6 +27,19 @@ type Options struct {
 	AlwaysMakeCrownjewel bool
 }
 
+// Apply applies options to the record metadata.
+func (o *Options) Apply(r record.Record) {
+	if r.Meta() == nil {
+		r.SetMeta(&record.Meta{})
+	}
+	if o.AlwaysMakeSecret {
+		r.Meta().MakeSecret()
+	}
+	if o.AlwaysMakeCrownjewel {
+		r.Meta().MakeCrownJewel()
+	}
+}
+
 // NewInterface returns a new Interface to the database.
 func NewInterface(opts *Options) *Interface {
 	if opts == nil {
@@ -61,7 +74,7 @@ func (i *Interface) getRecord(dbName string, dbKey string, check bool, mustBeWri
 		dbName, dbKey = record.ParseKey(dbKey)
 	}
 
-	db, err = getDatabase(dbName)
+	db, err = getController(dbName)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -72,6 +85,9 @@ func (i *Interface) getRecord(dbName string, dbKey string, check bool, mustBeWri
 
 	r, err = db.Get(dbKey)
 	if err != nil {
+		if err == ErrNotFound {
+			return nil, db, err
+		}
 		return nil, nil, err
 	}
 
@@ -108,26 +124,30 @@ func (i *Interface) InsertValue(key string, attribute string, value interface{})
 		return fmt.Errorf("failed to set value with %s: %s", acc.Type(), err)
 	}
 
+	i.options.Apply(r)
 	return db.Put(r)
 }
 
 // Put saves a record to the database.
 func (i *Interface) Put(r record.Record) error {
 	_, db, err := i.getRecord(r.DatabaseName(), r.DatabaseKey(), true, true)
-	if err != nil {
+	if err != nil && err != ErrNotFound {
 		return err
 	}
+
+	i.options.Apply(r)
 	return db.Put(r)
 }
 
-// PutNew saves a record to the database as a new record (ie. with a new creation timestamp).
+// PutNew saves a record to the database as a new record (ie. with new timestamps).
 func (i *Interface) PutNew(r record.Record) error {
 	_, db, err := i.getRecord(r.DatabaseName(), r.DatabaseKey(), true, true)
 	if err != nil && err != ErrNotFound {
 		return err
 	}
 
-	r.SetMeta(&record.Meta{})
+	i.options.Apply(r)
+	r.Meta().Reset()
 	return db.Put(r)
 }
 
@@ -141,6 +161,7 @@ func (i *Interface) SetAbsoluteExpiry(key string, time int64) error {
 	r.Lock()
 	defer r.Unlock()
 
+	i.options.Apply(r)
 	r.Meta().SetAbsoluteExpiry(time)
 	return db.Put(r)
 }
@@ -155,6 +176,7 @@ func (i *Interface) SetRelativateExpiry(key string, duration int64) error {
 	r.Lock()
 	defer r.Unlock()
 
+	i.options.Apply(r)
 	r.Meta().SetRelativateExpiry(duration)
 	return db.Put(r)
 }
@@ -169,6 +191,7 @@ func (i *Interface) MakeSecret(key string) error {
 	r.Lock()
 	defer r.Unlock()
 
+	i.options.Apply(r)
 	r.Meta().MakeSecret()
 	return db.Put(r)
 }
@@ -183,6 +206,7 @@ func (i *Interface) MakeCrownJewel(key string) error {
 	r.Lock()
 	defer r.Unlock()
 
+	i.options.Apply(r)
 	r.Meta().MakeCrownJewel()
 	return db.Put(r)
 }
@@ -197,13 +221,14 @@ func (i *Interface) Delete(key string) error {
 	r.Lock()
 	defer r.Unlock()
 
+	i.options.Apply(r)
 	r.Meta().Delete()
 	return db.Put(r)
 }
 
 // Query executes the given query on the database.
 func (i *Interface) Query(q *query.Query) (*iterator.Iterator, error) {
-	db, err := getDatabase(q.DatabaseName())
+	db, err := getController(q.DatabaseName())
 	if err != nil {
 		return nil, err
 	}
diff --git a/database/location.go b/database/location.go
index a44e415..0c095b6 100644
--- a/database/location.go
+++ b/database/location.go
@@ -1,42 +1,26 @@
 package database
 
 import (
-	"path"
-	"os"
-	"fmt"
 	"errors"
+	"fmt"
+	"os"
+	"path"
+)
+
+const (
+	databasesSubDir = "databases"
 )
 
 var (
 	rootDir string
 )
 
-// Initialize initialized the database
-func Initialize(location string) error {
-	if initialized.SetToIf(false, true) {
-		rootDir = location
-
-		err := checkRootDir()
-		if err != nil {
-			return fmt.Errorf("could not create/open database directory (%s): %s", rootDir, err)
-		}
-
-		err = loadRegistry()
-		if err != nil {
-			return fmt.Errorf("could not load database registry (%s): %s", path.Join(rootDir, registryFileName), err)
-		}
-
-		return nil
-	}
-	return errors.New("database already initialized")
-}
-
-func checkRootDir() error {
+func ensureDirectory(dirPath string) error {
 	// open dir
-	dir, err := os.Open(rootDir)
+	dir, err := os.Open(dirPath)
 	if err != nil {
-		if err == os.ErrNotExist {
-			return os.MkdirAll(rootDir, 0700)
+		if os.IsNotExist(err) {
+			return os.MkdirAll(dirPath, 0700)
 		}
 		return err
 	}
@@ -46,7 +30,9 @@ func checkRootDir() error {
 	if err != nil {
 		return err
 	}
-
+	if !fileInfo.IsDir() {
+		return errors.New("path exists and is not a directory")
+	}
 	if fileInfo.Mode().Perm() != 0700 {
 		return dir.Chmod(0700)
 	}
@@ -54,6 +40,13 @@ func checkRootDir() error {
 }
 
 // getLocation returns the storage location for the given name and type.
-func getLocation(name, storageType string) (location string, err error) {
-	return path.Join(rootDir, name, storageType), nil
+func getLocation(name, storageType string) (string, error) {
+	location := path.Join(rootDir, databasesSubDir, name, storageType)
+
+	// check location
+	err := ensureDirectory(location)
+	if err != nil {
+		return "", fmt.Errorf("location (%s) invalid: %s", location, err)
+	}
+	return location, nil
 }
diff --git a/database/main.go b/database/main.go
new file mode 100644
index 0000000..7bb781d
--- /dev/null
+++ b/database/main.go
@@ -0,0 +1,55 @@
+package database
+
+import (
+	"errors"
+	"fmt"
+	"path"
+
+	"github.com/tevino/abool"
+)
+
+var (
+	initialized = abool.NewBool(false)
+
+	shuttingDown   = abool.NewBool(false)
+	shutdownSignal = make(chan struct{})
+)
+
+// Initialize initialized the database
+func Initialize(location string) error {
+	if initialized.SetToIf(false, true) {
+		rootDir = location
+
+		err := ensureDirectory(rootDir)
+		if err != nil {
+			return fmt.Errorf("could not create/open database directory (%s): %s", rootDir, err)
+		}
+
+		err = loadRegistry()
+		if err != nil {
+			return fmt.Errorf("could not load database registry (%s): %s", path.Join(rootDir, registryFileName), err)
+		}
+
+		// start registry writer
+		go registryWriter()
+
+		return nil
+	}
+	return errors.New("database already initialized")
+}
+
+// Shutdown shuts down the whole database system.
+func Shutdown() (err error) {
+	if shuttingDown.SetToIf(false, true) {
+		close(shutdownSignal)
+	}
+
+	all := duplicateControllers()
+	for _, c := range all {
+		err = c.Shutdown()
+		if err != nil {
+			return
+		}
+	}
+	return
+}
diff --git a/database/maintainence.go b/database/maintainence.go
index 5b5f1f5..4029a69 100644
--- a/database/maintainence.go
+++ b/database/maintainence.go
@@ -14,8 +14,8 @@ func Maintain() (err error) {
 
 // MaintainThorough runs the MaintainThorough method on all storages.
 func MaintainThorough() (err error) {
-  controllers := duplicateControllers()
-  for _, c := range controllers {
+  all := duplicateControllers()
+  for _, c := range all {
     err = c.MaintainThorough()
     if err != nil {
       return
@@ -24,26 +24,12 @@ func MaintainThorough() (err error) {
   return
 }
 
-// Shutdown shuts down the whole database system.
-func Shutdown() (err error) {
-  shuttingDown.Set()
+func duplicateControllers() (all []*Controller) {
+  controllersLock.Lock()
+  defer controllersLock.Unlock()
 
-  controllers := duplicateControllers()
   for _, c := range controllers {
-    err = c.Shutdown()
-    if err != nil {
-      return
-    }
-  }
-  return
-}
-
-func duplicateControllers() (controllers []*Controller) {
-  databasesLock.Lock()
-  defer databasesLock.Unlock()
-
-  for _, c := range databases {
-    controllers = append(controllers, c)
+    all = append(all, c)
   }
 
   return
diff --git a/database/registry.go b/database/registry.go
index c83a9af..1322854 100644
--- a/database/registry.go
+++ b/database/registry.go
@@ -3,67 +3,100 @@ package database
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 	"regexp"
 	"sync"
+	"time"
 
 	"github.com/tevino/abool"
 )
 
-// RegisteredDatabase holds information about registered databases
-type RegisteredDatabase struct {
-	Name        string
-	Description string
-	StorageType string
-	PrimaryAPI  string
-}
-
-// Equal returns whether this instance equals another.
-func (r *RegisteredDatabase) Equal(o *RegisteredDatabase) bool {
-	if r.Name != o.Name ||
-		r.Description != o.Description ||
-		r.StorageType != o.StorageType ||
-		r.PrimaryAPI != o.PrimaryAPI {
-		return false
-	}
-	return true
-}
-
 const (
 	registryFileName = "databases.json"
 )
 
 var (
-	initialized = abool.NewBool(false)
+	writeRegistrySoon = abool.NewBool(false)
 
-	registry     map[string]*RegisteredDatabase
+	registry     map[string]*Database
 	registryLock sync.Mutex
 
-	nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{5,}$")
+	nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{4,}$")
 )
 
-// RegisterDatabase registers a new database.
-func RegisterDatabase(new *RegisteredDatabase) error {
+// Register registers a new database.
+// If the database is already registered, only
+// the description and the primary API will be
+// updated and the effective object will be returned.
+func Register(new *Database) (*Database, error) {
 	if !initialized.IsSet() {
-		return errors.New("database not initialized")
-	}
-
-	if !nameConstraint.MatchString(new.Name) {
-		return errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 5 characters long")
+		return nil, errors.New("database not initialized")
 	}
 
 	registryLock.Lock()
 	defer registryLock.Unlock()
 
 	registeredDB, ok := registry[new.Name]
-	if !ok || !new.Equal(registeredDB) {
+	save := false
+
+	if ok {
+		// update database
+		if registeredDB.Description != new.Description {
+			registeredDB.Description = new.Description
+			save = true
+		}
+		if registeredDB.PrimaryAPI != new.PrimaryAPI {
+			registeredDB.PrimaryAPI = new.PrimaryAPI
+			save = true
+		}
+	} else {
+		// register new database
+		if !nameConstraint.MatchString(new.Name) {
+			return nil, errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 4 characters long")
+		}
+
+		now := time.Now().Round(time.Second)
+		new.Registered = now
+		new.LastUpdated = now
+		new.LastLoaded = time.Time{}
+
 		registry[new.Name] = new
-		return saveRegistry()
+		save = true
 	}
 
-	return nil
+	if save {
+		if ok {
+			registeredDB.Updated()
+		}
+		err := saveRegistry(false)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ok {
+		return registeredDB, nil
+	}
+	return nil, nil
+}
+
+func getDatabase(name string) (*Database, error) {
+	registryLock.Lock()
+	defer registryLock.Unlock()
+
+	registeredDB, ok := registry[name]
+	if !ok {
+		return nil, fmt.Errorf(`database "%s" not registered`, name)
+	}
+	if time.Now().Add(-24 * time.Hour).After(registeredDB.LastLoaded) {
+		writeRegistrySoon.Set()
+	}
+	registeredDB.Loaded()
+
+	return registeredDB, nil
 }
 
 func loadRegistry() error {
@@ -74,15 +107,15 @@ func loadRegistry() error {
 	filePath := path.Join(rootDir, registryFileName)
 	data, err := ioutil.ReadFile(filePath)
 	if err != nil {
-		if err == os.ErrNotExist {
-			registry = make(map[string]*RegisteredDatabase)
+		if os.IsNotExist(err) {
+			registry = make(map[string]*Database)
 			return nil
 		}
 		return err
 	}
 
 	// parse
-	new := make(map[string]*RegisteredDatabase)
+	new := make(map[string]*Database)
 	err = json.Unmarshal(data, new)
 	if err != nil {
 		return err
@@ -93,12 +126,14 @@ func loadRegistry() error {
 	return nil
 }
 
-func saveRegistry() error {
-	registryLock.Lock()
-	defer registryLock.Unlock()
+func saveRegistry(lock bool) error {
+	if lock {
+		registryLock.Lock()
+		defer registryLock.Unlock()
+	}
 
 	// marshal
-	data, err := json.Marshal(registry)
+	data, err := json.MarshalIndent(registry, "", "\t")
 	if err != nil {
 		return err
 	}
@@ -107,3 +142,17 @@ func saveRegistry() error {
 	filePath := path.Join(rootDir, registryFileName)
 	return ioutil.WriteFile(filePath, data, 0600)
 }
+
+func registryWriter() {
+	for {
+		select {
+		case <-time.After(1 * time.Hour):
+			if writeRegistrySoon.SetToIf(true, false) {
+				saveRegistry(true)
+			}
+		case <-shutdownSignal:
+			saveRegistry(true)
+			return
+		}
+	}
+}
diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
index af04bf4..ebe849a 100644
--- a/database/storage/badger/badger.go
+++ b/database/storage/badger/badger.go
@@ -116,7 +116,7 @@ func (b *Badger) Delete(key string) error {
 }
 
 // Query returns a an iterator for the supplied query.
-func (b *Badger) Query(q *query.Query) (*iterator.Iterator, error) {
+func (b *Badger) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {
 	return nil, errors.New("query not implemented by badger")
 }
 
diff --git a/database/storage/badger/badger_test.go b/database/storage/badger/badger_test.go
new file mode 100644
index 0000000..cc0f2f8
--- /dev/null
+++ b/database/storage/badger/badger_test.go
@@ -0,0 +1,69 @@
+package badger
+
+import (
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+
+	"github.com/Safing/portbase/database/record"
+)
+
+type TestRecord struct {
+	record.Base
+	lock sync.Mutex
+	S    string
+	I    int
+	I8   int8
+	I16  int16
+	I32  int32
+	I64  int64
+	UI   uint
+	UI8  uint8
+	UI16 uint16
+	UI32 uint32
+	UI64 uint64
+	F32  float32
+	F64  float64
+	B    bool
+}
+
+func (tr *TestRecord) Lock() {
+}
+
+func (tr *TestRecord) Unlock() {
+}
+
+func TestBadger(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "testing-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(testDir) // clean up
+
+	db, err := NewBadger("test", testDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a := &TestRecord{S: "banana"}
+	a.SetMeta(&record.Meta{})
+	a.Meta().Update()
+	a.SetKey("test:A")
+
+	err = db.Put(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	r1, err := db.Get("A")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1 := r1.(*TestRecord)
+
+	if a.S != a1.S {
+		t.Fatal("mismatch")
+	}
+}
diff --git a/database/storage/errors.go b/database/storage/errors.go
index 91c7689..a280296 100644
--- a/database/storage/errors.go
+++ b/database/storage/errors.go
@@ -4,5 +4,5 @@ import "errors"
 
 // Errors for storages
 var (
-	ErrNotFound = errors.New("not found")
+	ErrNotFound = errors.New("storage entry could not be found")
 )
diff --git a/database/storage/storages.go b/database/storage/storages.go
index 948af83..1fa7448 100644
--- a/database/storage/storages.go
+++ b/database/storage/storages.go
@@ -10,7 +10,7 @@ import (
 type Factory func(name, location string) (Interface, error)
 
 var (
-	storages     map[string]Factory
+	storages     = make(map[string]Factory)
 	storagesLock sync.Mutex
 )
 
@@ -38,9 +38,9 @@ func StartDatabase(name, storageType, location string) (Interface, error) {
 	storagesLock.Lock()
 	defer storagesLock.Unlock()
 
-	factory, ok := storages[name]
+	factory, ok := storages[storageType]
 	if !ok {
-		return nil, fmt.Errorf("storage of this type (%s) does not exist", storageType)
+		return nil, fmt.Errorf("storage type %s not registered", storageType)
 	}
 
 	return factory(name, location)

From 014f944b2a5256ad2ab13dc740639b2e9782df53 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 11 Sep 2018 18:59:48 +0200
Subject: [PATCH 17/20] Mock config stuff to do

---
 config/integration/api.go         | 3 +++
 config/integration/module.go      | 3 +++
 config/integration/persistence.go | 4 ++++
 config/registry.go                | 2 +-
 4 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 config/integration/api.go
 create mode 100644 config/integration/module.go
 create mode 100644 config/integration/persistence.go

diff --git a/config/integration/api.go b/config/integration/api.go
new file mode 100644
index 0000000..e6aa1b2
--- /dev/null
+++ b/config/integration/api.go
@@ -0,0 +1,3 @@
+package integration
+
+// API
diff --git a/config/integration/module.go b/config/integration/module.go
new file mode 100644
index 0000000..c9bba1b
--- /dev/null
+++ b/config/integration/module.go
@@ -0,0 +1,3 @@
+package integration
+
+// register as module
diff --git a/config/integration/persistence.go b/config/integration/persistence.go
new file mode 100644
index 0000000..09cd938
--- /dev/null
+++ b/config/integration/persistence.go
@@ -0,0 +1,4 @@
+package integration
+
+// persist config file
+// create callback function in config to get updates
diff --git a/config/registry.go b/config/registry.go
index b67c981..a998987 100644
--- a/config/registry.go
+++ b/config/registry.go
@@ -7,7 +7,7 @@ import (
 	"sync"
 )
 
-// Variable Type IDs for frontend Identification. Values over 100 are free for custom use.
+// Variable Type IDs for frontend Identification. Values from 100 are free for custom use.
 const (
 	OptTypeString      uint8 = 1
 	OptTypeStringArray uint8 = 2

From 1c7f98d9bae9b9582a480b2a81daa853936341f7 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 12 Sep 2018 17:10:35 +0200
Subject: [PATCH 18/20] Work on tests and query

---
 database/controller.go                 |   8 +-
 database/database_test.go              |  36 +++++++-
 database/dbmodule/maintenance.go       |  14 ++--
 database/maintainence.go               |  36 --------
 database/maintenance.go                |  92 +++++++++++++++++++++
 database/record/base.go                |   8 +-
 database/record/meta.go                |   4 +-
 database/record/record.go              |   4 +-
 database/record/wrapper.go             |   6 +-
 database/record/wrapper_test.go        |   4 +-
 database/storage/badger/badger.go      | 110 +++++++++++++++++++------
 database/storage/badger/badger_test.go |  53 +++++++++++-
 database/storage/interface.go          |   1 +
 13 files changed, 288 insertions(+), 88 deletions(-)
 delete mode 100644 database/maintainence.go
 create mode 100644 database/maintenance.go

diff --git a/database/controller.go b/database/controller.go
index 2c4875f..e1325a4 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -2,7 +2,6 @@ package database
 
 import (
 	"sync"
-	"time"
 
 	"github.com/tevino/abool"
 
@@ -35,6 +34,11 @@ func (c *Controller) ReadOnly() bool {
 	return c.storage.ReadOnly()
 }
 
+// Injected returns whether the storage is injected.
+func (c *Controller) Injected() bool {
+	return c.storage.Injected()
+}
+
 // Get return the record with the given key.
 func (c *Controller) Get(key string) (record.Record, error) {
 	if shuttingDown.IsSet() {
@@ -53,7 +57,7 @@ func (c *Controller) Get(key string) (record.Record, error) {
 	r.Lock()
 	defer r.Unlock()
 
-	if !r.Meta().CheckValidity(time.Now().Unix()) {
+	if !r.Meta().CheckValidity() {
 		return nil, ErrNotFound
 	}
 
diff --git a/database/database_test.go b/database/database_test.go
index bab7579..9cd679d 100644
--- a/database/database_test.go
+++ b/database/database_test.go
@@ -38,8 +38,8 @@ func (tr *TestRecord) Lock() {
 func (tr *TestRecord) Unlock() {
 }
 
-func makeKey(storageType, key string) string {
-	return fmt.Sprintf("%s:%s", storageType, key)
+func makeKey(dbName, key string) string {
+	return fmt.Sprintf("%s:%s", dbName, key)
 }
 
 func testDatabase(t *testing.T, storageType string) {
@@ -56,13 +56,38 @@ func testDatabase(t *testing.T, storageType string) {
 
 	db := NewInterface(nil)
 
-	new := &TestRecord{}
+	new := &TestRecord{
+		S:    "banana",
+		I:    42,
+		I8:   42,
+		I16:  42,
+		I32:  42,
+		I64:  42,
+		UI:   42,
+		UI8:  42,
+		UI16: 42,
+		UI32: 42,
+		UI64: 42,
+		F32:  42.42,
+		F64:  42.42,
+		B:    true,
+	}
+	new.SetMeta(&record.Meta{})
+	new.Meta().Update()
 	new.SetKey(makeKey(dbName, "A"))
 	err = db.Put(new)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	exists, err := db.Exists(makeKey(dbName, "A"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !exists {
+		t.Fatalf("record %s should exist!", makeKey(dbName, "A"))
+	}
+
 	_, err = db.Get(makeKey(dbName, "A"))
 	if err != nil {
 		t.Fatal(err)
@@ -92,6 +117,11 @@ func TestDatabaseSystem(t *testing.T) {
 
 	testDatabase(t, "badger")
 
+	err = MaintainRecordStates()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	err = Maintain()
 	if err != nil {
 		t.Fatal(err)
diff --git a/database/dbmodule/maintenance.go b/database/dbmodule/maintenance.go
index df9a9eb..4bdd6fe 100644
--- a/database/dbmodule/maintenance.go
+++ b/database/dbmodule/maintenance.go
@@ -8,8 +8,8 @@ import (
 )
 
 func maintainer() {
-  ticker := time.NewTicker(1 * time.Hour)
-  tickerThorough := time.NewTicker(10 * time.Minute)
+  ticker := time.NewTicker(10 * time.Minute)
+  longTicker := time.NewTicker(1 * time.Hour)
   maintenanceWg.Add(1)
 
   for {
@@ -19,10 +19,14 @@ func maintainer() {
       if err != nil {
         log.Errorf("database: maintenance error: %s", err)
       }
-    case <- ticker.C:
-      err := database.MaintainThorough()
+    case <- longTicker.C:
+      err := database.MaintainRecordStates()
       if err != nil {
-        log.Errorf("database: maintenance (thorough) error: %s", err)
+        log.Errorf("database: record states maintenance error: %s", err)
+      }
+      err = database.MaintainThorough()
+      if err != nil {
+        log.Errorf("database: thorough maintenance error: %s", err)
       }
     case <-shutdownSignal:
       maintenanceWg.Done()
diff --git a/database/maintainence.go b/database/maintainence.go
deleted file mode 100644
index 4029a69..0000000
--- a/database/maintainence.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package database
-
-// Maintain runs the Maintain method on all storages.
-func Maintain() (err error) {
-  controllers := duplicateControllers()
-  for _, c := range controllers {
-    err = c.Maintain()
-    if err != nil {
-      return
-    }
-  }
-  return
-}
-
-// MaintainThorough runs the MaintainThorough method on all storages.
-func MaintainThorough() (err error) {
-  all := duplicateControllers()
-  for _, c := range all {
-    err = c.MaintainThorough()
-    if err != nil {
-      return
-    }
-  }
-  return
-}
-
-func duplicateControllers() (all []*Controller) {
-  controllersLock.Lock()
-  defer controllersLock.Unlock()
-
-  for _, c := range controllers {
-    all = append(all, c)
-  }
-
-  return
-}
diff --git a/database/maintenance.go b/database/maintenance.go
new file mode 100644
index 0000000..d1399bd
--- /dev/null
+++ b/database/maintenance.go
@@ -0,0 +1,92 @@
+package database
+
+import (
+  "time"
+
+  "github.com/Safing/portbase/database/query"
+  "github.com/Safing/portbase/database/record"
+)
+
+// Maintain runs the Maintain method on all storages.
+func Maintain() (err error) {
+  controllers := duplicateControllers()
+  for _, c := range controllers {
+    err = c.Maintain()
+    if err != nil {
+      return
+    }
+  }
+  return
+}
+
+// MaintainThorough runs the MaintainThorough method on all storages.
+func MaintainThorough() (err error) {
+  all := duplicateControllers()
+  for _, c := range all {
+    err = c.MaintainThorough()
+    if err != nil {
+      return
+    }
+  }
+  return
+}
+
+// MaintainRecordStates runs record state lifecycle maintenance on all storages.
+func MaintainRecordStates() error {
+  all := duplicateControllers()
+  now := time.Now().Unix()
+  thirtyDaysAgo := time.Now().Add(-30*24*time.Hour).Unix()
+
+  for _, c := range all {
+
+    if c.ReadOnly() || c.Injected() {
+      continue
+    }
+
+    q, err := query.New("").Check()
+    if err != nil {
+      return err
+    }
+
+    it, err := c.Query(q, true, true)
+    if err != nil {
+      return err
+    }
+
+    var toDelete []record.Record
+    var toExpire []record.Record
+
+    for r := range it.Next {
+      switch {
+      case r.Meta().Deleted < thirtyDaysAgo:
+        toDelete = append(toDelete, r)
+      case r.Meta().Expires < now:
+        toExpire = append(toExpire, r)
+      }
+    }
+    if it.Error != nil {
+      return err
+    }
+
+    for _, r := range toDelete {
+      c.storage.Delete(r.DatabaseKey())
+    }
+    for _, r := range toExpire {
+      r.Meta().Delete()
+      return c.Put(r)
+    }
+
+  }
+  return nil
+}
+
+func duplicateControllers() (all []*Controller) {
+  controllersLock.Lock()
+  defer controllersLock.Unlock()
+
+  for _, c := range controllers {
+    all = append(all, c)
+  }
+
+  return
+}
diff --git a/database/record/base.go b/database/record/base.go
index c2966bb..44d9b54 100644
--- a/database/record/base.go
+++ b/database/record/base.go
@@ -52,7 +52,7 @@ func (b *Base) SetMeta(meta *Meta) {
 }
 
 // Marshal marshals the object, without the database key or metadata
-func (b *Base) Marshal(format uint8) ([]byte, error) {
+func (b *Base) Marshal(self Record, format uint8) ([]byte, error) {
 	if b.Meta() == nil {
 		return nil, errors.New("missing meta")
 	}
@@ -61,7 +61,7 @@ func (b *Base) Marshal(format uint8) ([]byte, error) {
 		return nil, nil
 	}
 
-	dumped, err := dsd.Dump(b, format)
+	dumped, err := dsd.Dump(self, format)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +69,7 @@ func (b *Base) Marshal(format uint8) ([]byte, error) {
 }
 
 // MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
-func (b *Base) MarshalRecord() ([]byte, error) {
+func (b *Base) MarshalRecord(self Record) ([]byte, error) {
 	if b.Meta() == nil {
 		return nil, errors.New("missing meta")
 	}
@@ -85,7 +85,7 @@ func (b *Base) MarshalRecord() ([]byte, error) {
 	c.AppendAsBlock(metaSection)
 
 	// data
-	dataSection, err := b.Marshal(dsd.JSON)
+	dataSection, err := b.Marshal(self, dsd.JSON)
 	if err != nil {
 		return nil, err
 	}
diff --git a/database/record/meta.go b/database/record/meta.go
index d916816..a2c8bb4 100644
--- a/database/record/meta.go
+++ b/database/record/meta.go
@@ -79,11 +79,11 @@ func (m *Meta) Delete() {
 }
 
 // CheckValidity checks whether the database record is valid.
-func (m *Meta) CheckValidity(now int64) (valid bool) {
+func (m *Meta) CheckValidity() (valid bool) {
 	switch {
 	case m.Deleted > 0:
 		return false
-	case m.Expires < now:
+	case m.Expires > 0 && m.Expires < time.Now().Unix():
 		return false
 	default:
 		return true
diff --git a/database/record/record.go b/database/record/record.go
index c40fca2..a09de4b 100644
--- a/database/record/record.go
+++ b/database/record/record.go
@@ -11,8 +11,8 @@ type Record interface {
 	Meta() *Meta
 	SetMeta(meta *Meta)
 
-	Marshal(format uint8) ([]byte, error)
-	MarshalRecord() ([]byte, error)
+	Marshal(r Record, format uint8) ([]byte, error)
+	MarshalRecord(r Record) ([]byte, error)
 
 	Lock()
 	Unlock()
diff --git a/database/record/wrapper.go b/database/record/wrapper.go
index d6352b7..6e7ef69 100644
--- a/database/record/wrapper.go
+++ b/database/record/wrapper.go
@@ -74,7 +74,7 @@ func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
 }
 
 // Marshal marshals the object, without the database key or metadata
-func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
+func (w *Wrapper) Marshal(r Record, storageType uint8) ([]byte, error) {
 	if w.Meta() == nil {
 		return nil, errors.New("missing meta")
 	}
@@ -90,7 +90,7 @@ func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
 }
 
 // MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
-func (w *Wrapper) MarshalRecord() ([]byte, error) {
+func (w *Wrapper) MarshalRecord(r Record) ([]byte, error) {
 	// Duplication necessary, as the version from Base would call Base.Marshal instead of Wrapper.Marshal
 
 	if w.Meta() == nil {
@@ -108,7 +108,7 @@ func (w *Wrapper) MarshalRecord() ([]byte, error) {
 	c.AppendAsBlock(metaSection)
 
 	// data
-	dataSection, err := w.Marshal(dsd.JSON)
+	dataSection, err := w.Marshal(r, dsd.JSON)
 	if err != nil {
 		return nil, err
 	}
diff --git a/database/record/wrapper_test.go b/database/record/wrapper_test.go
index e725916..e2988f0 100644
--- a/database/record/wrapper_test.go
+++ b/database/record/wrapper_test.go
@@ -30,7 +30,7 @@ func TestWrapper(t *testing.T) {
 		t.Error("data mismatch")
 	}
 
-	encoded, err := wrapper.Marshal(dsd.JSON)
+	encoded, err := wrapper.Marshal(wrapper, dsd.JSON)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -39,7 +39,7 @@ func TestWrapper(t *testing.T) {
 	}
 
 	wrapper.SetMeta(&Meta{})
-	raw, err := wrapper.MarshalRecord()
+	raw, err := wrapper.MarshalRecord(wrapper)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
index ebe849a..f685d90 100644
--- a/database/storage/badger/badger.go
+++ b/database/storage/badger/badger.go
@@ -2,9 +2,12 @@ package badger
 
 import (
 	"errors"
+	"fmt"
+	"time"
 
 	"github.com/dgraph-io/badger"
 
+	"github.com/Safing/portbase/database/accessor"
 	"github.com/Safing/portbase/database/iterator"
 	"github.com/Safing/portbase/database/query"
 	"github.com/Safing/portbase/database/record"
@@ -38,24 +41,6 @@ func NewBadger(name, location string) (storage.Interface, error) {
 	}, nil
 }
 
-// Exists returns whether an entry with the given key exists.
-func (b *Badger) Exists(key string) (bool, error) {
-	err := b.db.View(func(txn *badger.Txn) error {
-		_, err := txn.Get([]byte(key))
-		if err != nil {
-			if err == badger.ErrKeyNotFound {
-				return nil
-			}
-			return err
-		}
-		return nil
-	})
-	if err == nil {
-		return true, nil
-	}
-	return false, nil
-}
-
 // Get returns a database record.
 func (b *Badger) Get(key string) (record.Record, error) {
 	var item *badger.Item
@@ -75,9 +60,10 @@ func (b *Badger) Get(key string) (record.Record, error) {
 		return nil, err
 	}
 
-	if item.IsDeletedOrExpired() {
-		return nil, storage.ErrNotFound
-	}
+	// DO NOT check for this, as we got our own machanism for that.
+	// if item.IsDeletedOrExpired() {
+	// 	return nil, storage.ErrNotFound
+	// }
 
 	data, err := item.ValueCopy(nil)
 	if err != nil {
@@ -92,14 +78,14 @@ func (b *Badger) Get(key string) (record.Record, error) {
 }
 
 // Put stores a record in the database.
-func (b *Badger) Put(m record.Record) error {
-	data, err := m.MarshalRecord()
+func (b *Badger) Put(r record.Record) error {
+	data, err := r.MarshalRecord(r)
 	if err != nil {
 		return err
 	}
 
 	err = b.db.Update(func(txn *badger.Txn) error {
-		return txn.Set([]byte(m.DatabaseKey()), data)
+		return txn.Set([]byte(r.DatabaseKey()), data)
 	})
 	return err
 }
@@ -117,7 +103,76 @@ func (b *Badger) Delete(key string) error {
 
 // Query returns a an iterator for the supplied query.
 func (b *Badger) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {
-	return nil, errors.New("query not implemented by badger")
+	_, err := q.Check()
+	if err != nil {
+		return nil, fmt.Errorf("invalid query: %s", err)
+	}
+
+	queryIter := iterator.New()
+
+	go b.queryExecutor(queryIter, q, local, internal)
+	return queryIter, nil
+}
+
+func (b *Badger) queryExecutor(queryIter *iterator.Iterator, q *query.Query, local, internal bool) {
+	err := b.db.View(func(txn *badger.Txn) error {
+		it := txn.NewIterator(badger.DefaultIteratorOptions)
+		defer it.Close()
+		prefix := []byte(q.DatabaseKeyPrefix())
+		for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+			item := it.Item()
+
+			data, err := item.Value()
+			if err != nil {
+				return err
+			}
+
+			r, err := record.NewRawWrapper(b.name, string(item.Key()), data)
+			if err != nil {
+				return err
+			}
+
+			if !r.Meta().CheckValidity() {
+				continue
+			}
+			if !r.Meta().CheckPermission(local, internal) {
+				continue
+			}
+
+			if len(r.Data) > 1 {
+				jsonData := r.Data[1:]
+				acc := accessor.NewJSONBytesAccessor(&jsonData)
+				if q.Matches(acc) {
+
+					copiedData, err := item.ValueCopy(nil)
+					if err != nil {
+						return err
+					}
+					new, err := record.NewRawWrapper(b.name, string(item.Key()), copiedData)
+					if err != nil {
+						return err
+					}
+					select {
+					case queryIter.Next <- new:
+					default:
+						select {
+						case queryIter.Next <- new:
+						case <-time.After(1 * time.Minute):
+							return errors.New("query timeout")
+						}
+					}
+
+				}
+			}
+
+		}
+		return nil
+	})
+
+	if err != nil {
+		queryIter.Error = err
+	}
+	close(queryIter.Next)
 }
 
 // ReadOnly returns whether the database is read only.
@@ -125,6 +180,11 @@ func (b *Badger) ReadOnly() bool {
 	return false
 }
 
+// Injected returns whether the database is injected.
+func (b *Badger) Injected() bool {
+	return false
+}
+
 // Maintain runs a light maintenance operation on the database.
 func (b *Badger) Maintain() error {
 	b.db.RunValueLogGC(0.7)
diff --git a/database/storage/badger/badger_test.go b/database/storage/badger/badger_test.go
index cc0f2f8..ee22fb2 100644
--- a/database/storage/badger/badger_test.go
+++ b/database/storage/badger/badger_test.go
@@ -3,6 +3,7 @@ package badger
 import (
 	"io/ioutil"
 	"os"
+	"reflect"
 	"sync"
 	"testing"
 
@@ -46,7 +47,22 @@ func TestBadger(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	a := &TestRecord{S: "banana"}
+	a := &TestRecord{
+		S:    "banana",
+		I:    42,
+		I8:   42,
+		I16:  42,
+		I32:  42,
+		I64:  42,
+		UI:   42,
+		UI8:  42,
+		UI16: 42,
+		UI32: 42,
+		UI64: 42,
+		F32:  42.42,
+		F64:  42.42,
+		B:    true,
+	}
 	a.SetMeta(&record.Meta{})
 	a.Meta().Update()
 	a.SetKey("test:A")
@@ -61,9 +77,38 @@ func TestBadger(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	a1 := r1.(*TestRecord)
+	a1 := &TestRecord{}
+	_, err = record.Unwrap(r1, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	if a.S != a1.S {
-		t.Fatal("mismatch")
+	if !reflect.DeepEqual(a, a1) {
+		t.Fatalf("mismatch, got %v", a1)
+	}
+
+	err = db.Delete("A")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.Get("A")
+	if err == nil {
+		t.Fatal("should fail")
+	}
+
+	err = db.Maintain()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.MaintainThorough()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Shutdown()
+	if err != nil {
+		t.Fatal(err)
 	}
 }
diff --git a/database/storage/interface.go b/database/storage/interface.go
index b5bd803..73b1a5f 100644
--- a/database/storage/interface.go
+++ b/database/storage/interface.go
@@ -14,6 +14,7 @@ type Interface interface {
 	Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
 
 	ReadOnly() bool
+	Injected() bool
 	Maintain() error
 	MaintainThorough() error
 	Shutdown() error

From 2c704168732593887c1cc64232752345d6dec17b Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 13 Sep 2018 21:54:20 +0200
Subject: [PATCH 19/20] Start work on hooks and subscriptions

---
 database/boilerplate_test.go           |  64 ++++++++++++
 database/controller.go                 |  80 ++++++++++++++-
 database/database_test.go              |  94 +++++++++---------
 database/doc.go                        | 129 +++++++++----------------
 database/hook.go                       |  72 ++++++++++++++
 database/hookbase.go                   |  29 ++++++
 database/iterator/iterator.go          |   2 +
 database/query/query.go                |  25 ++++-
 database/record/base.go                |   6 ++
 database/record/record.go              |   9 +-
 database/record/wrapper.go             |  51 ++++++----
 database/storage/badger/badger.go      |  37 +++----
 database/storage/badger/badger_test.go |   2 +-
 database/subscription.go               |  59 +++++++++++
 14 files changed, 482 insertions(+), 177 deletions(-)
 create mode 100644 database/boilerplate_test.go
 create mode 100644 database/hook.go
 create mode 100644 database/hookbase.go
 create mode 100644 database/subscription.go

diff --git a/database/boilerplate_test.go b/database/boilerplate_test.go
new file mode 100644
index 0000000..3fd5dac
--- /dev/null
+++ b/database/boilerplate_test.go
@@ -0,0 +1,64 @@
+package database
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/Safing/portbase/database/record"
+)
+
+type Example struct {
+	record.Base
+	sync.Mutex
+
+	Name  string
+	Score int
+}
+
+var (
+	exampleDB = NewInterface(nil)
+)
+
+// GetExample gets an Example from the database.
+func GetExample(key string) (*Example, error) {
+	r, err := exampleDB.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// unwrap
+	if r.IsWrapped() {
+		// only allocate a new struct, if we need it
+		new := &Example{}
+		err = record.Unwrap(r, new)
+		if err != nil {
+			return nil, err
+		}
+		return new, nil
+	}
+
+	// or adjust type
+	new, ok := r.(*Example)
+	if !ok {
+		return nil, fmt.Errorf("record not of type *Example, but %T", r)
+	}
+	return new, nil
+}
+
+func (e *Example) Save() error {
+	return exampleDB.Put(e)
+}
+
+func (e *Example) SaveAs(key string) error {
+	e.SetKey(key)
+	return exampleDB.PutNew(e)
+}
+
+func NewExample(key, name string, score int) *Example {
+	new := &Example{
+		Name:  name,
+		Score: score,
+	}
+	new.SetKey(key)
+	return new
+}
diff --git a/database/controller.go b/database/controller.go
index e1325a4..d2f279c 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -14,6 +14,10 @@ import (
 // A Controller takes care of all the extra database logic.
 type Controller struct {
 	storage   storage.Interface
+
+	hooks []*RegisteredHook
+	subscriptions []*Subscription
+
 	writeLock sync.RWMutex
 	readLock  sync.RWMutex
 	migrating *abool.AtomicBool // TODO
@@ -45,6 +49,19 @@ func (c *Controller) Get(key string) (record.Record, error) {
 		return nil, ErrShuttingDown
 	}
 
+	c.readLock.RLock()
+	defer c.readLock.RUnlock()
+
+	// process hooks
+	for _, hook := range c.hooks {
+		if hook.q.MatchesKey(key) {
+			err := hook.hook.PreGet(key)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	r, err := c.storage.Get(key)
 	if err != nil {
 		// replace not found error
@@ -57,6 +74,16 @@ func (c *Controller) Get(key string) (record.Record, error) {
 	r.Lock()
 	defer r.Unlock()
 
+	// process hooks
+	for _, hook := range c.hooks {
+		if hook.q.Matches(r) {
+			r, err = hook.hook.PostGet(r)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	if !r.Meta().CheckValidity() {
 		return nil, ErrNotFound
 	}
@@ -65,7 +92,7 @@ func (c *Controller) Get(key string) (record.Record, error) {
 }
 
 // Put saves a record in the database.
-func (c *Controller) Put(r record.Record) error {
+func (c *Controller) Put(r record.Record) (err error) {
 	if shuttingDown.IsSet() {
 		return ErrShuttingDown
 	}
@@ -74,12 +101,40 @@ func (c *Controller) Put(r record.Record) error {
 		return ErrReadOnly
 	}
 
+	r.Lock()
+	defer r.Unlock()
+
+	// process hooks
+	for _, hook := range c.hooks {
+		if hook.q.Matches(r) {
+		r, err = hook.hook.PrePut(r)
+		if err != nil {
+			return err
+		}
+	}
+	}
+
 	if r.Meta() == nil {
 		r.SetMeta(&record.Meta{})
 	}
 	r.Meta().Update()
 
-	return c.storage.Put(r)
+	c.writeLock.RLock()
+	defer c.writeLock.RUnlock()
+
+	err = c.storage.Put(r)
+	if err != nil {
+		return err
+	}
+
+	// process hooks
+	for _, hook := range c.hooks {
+		if hook.q.Matches(r) {
+			hook.hook.PostPut(r)
+		}
+	}
+
+	return nil
 }
 
 // Query executes the given query on the database.
@@ -87,20 +142,39 @@ func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iter
 	if shuttingDown.IsSet() {
 		return nil, ErrShuttingDown
 	}
-	return c.storage.Query(q, local, internal)
+
+	c.readLock.RLock()
+	it, err := c.storage.Query(q, local, internal)
+	if err != nil {
+		c.readLock.RUnlock()
+		return nil, err
+	}
+
+	go c.readUnlockerAfterQuery(it)
+	return it, nil
+}
+
+func (c *Controller) readUnlockerAfterQuery(it *iterator.Iterator) {
+	<- it.Done
+	c.readLock.RUnlock()
 }
 
 // Maintain runs the Maintain method no the storage.
 func (c *Controller) Maintain() error {
+	c.writeLock.RLock()
+	defer c.writeLock.RUnlock()
 	return c.storage.Maintain()
 }
 
 // MaintainThorough runs the MaintainThorough method no the storage.
 func (c *Controller) MaintainThorough() error {
+	c.writeLock.RLock()
+	defer c.writeLock.RUnlock()
 	return c.storage.MaintainThorough()
 }
 
 // Shutdown shuts down the storage.
 func (c *Controller) Shutdown() error {
+	// TODO: should we wait for gets/puts/queries to complete?
 	return c.storage.Shutdown()
 }
diff --git a/database/database_test.go b/database/database_test.go
index 9cd679d..d38f575 100644
--- a/database/database_test.go
+++ b/database/database_test.go
@@ -3,41 +3,17 @@ package database
 import (
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
+	"reflect"
 	"runtime/pprof"
-	"sync"
 	"testing"
 	"time"
 
-	"github.com/Safing/portbase/database/record"
+	q "github.com/Safing/portbase/database/query"
 	_ "github.com/Safing/portbase/database/storage/badger"
 )
 
-type TestRecord struct {
-	record.Base
-	lock sync.Mutex
-	S    string
-	I    int
-	I8   int8
-	I16  int16
-	I32  int32
-	I64  int64
-	UI   uint
-	UI8  uint8
-	UI16 uint16
-	UI32 uint32
-	UI64 uint64
-	F32  float32
-	F64  float64
-	B    bool
-}
-
-func (tr *TestRecord) Lock() {
-}
-
-func (tr *TestRecord) Unlock() {
-}
-
 func makeKey(dbName, key string) string {
 	return fmt.Sprintf("%s:%s", dbName, key)
 }
@@ -56,26 +32,20 @@ func testDatabase(t *testing.T, storageType string) {
 
 	db := NewInterface(nil)
 
-	new := &TestRecord{
-		S:    "banana",
-		I:    42,
-		I8:   42,
-		I16:  42,
-		I32:  42,
-		I64:  42,
-		UI:   42,
-		UI8:  42,
-		UI16: 42,
-		UI32: 42,
-		UI64: 42,
-		F32:  42.42,
-		F64:  42.42,
-		B:    true,
+	A := NewExample(makeKey(dbName, "A"), "Herbert", 411)
+	err = A.Save()
+	if err != nil {
+		t.Fatal(err)
 	}
-	new.SetMeta(&record.Meta{})
-	new.Meta().Update()
-	new.SetKey(makeKey(dbName, "A"))
-	err = db.Put(new)
+
+	B := NewExample(makeKey(dbName, "B"), "Fritz", 347)
+	err = B.Save()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	C := NewExample(makeKey(dbName, "C"), "Norbert", 217)
+	err = C.Save()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -88,10 +58,40 @@ func testDatabase(t *testing.T, storageType string) {
 		t.Fatalf("record %s should exist!", makeKey(dbName, "A"))
 	}
 
-	_, err = db.Get(makeKey(dbName, "A"))
+	A1, err := GetExample(makeKey(dbName, "A"))
 	if err != nil {
 		t.Fatal(err)
 	}
+	if !reflect.DeepEqual(A, A1) {
+		log.Fatalf("A and A1 mismatch, A1: %v", A1)
+	}
+
+	query, err := q.New(dbName).Where(
+		q.And(
+			q.Where("Name", q.EndsWith, "bert"),
+			q.Where("Score", q.GreaterThan, 100),
+		),
+	).Check()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	it, err := db.Query(query)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cnt := 0
+	for _ = range it.Next {
+		cnt++
+	}
+	if it.Error != nil {
+		t.Fatal(it.Error)
+	}
+	if cnt != 2 {
+		t.Fatal("expected two records")
+	}
+
 }
 
 func TestDatabaseSystem(t *testing.T) {
diff --git a/database/doc.go b/database/doc.go
index f5c0a9c..d93ff5f 100644
--- a/database/doc.go
+++ b/database/doc.go
@@ -3,98 +3,63 @@
 /*
 Package database provides a universal interface for interacting with the database.
 
-The Lazy Database
+A Lazy Database
 
 The database system can handle Go structs as well as serialized data by the dsd package.
 While data is in transit within the system, it does not know which form it currently has. Only when it reaches its destination, it must ensure that it is either of a certain type or dump it.
 
-Internals
+Record Interface
 
-The database system uses the Model interface to transparently handle all types of structs that get saved in the database. Structs include Base struct to fulfill most parts of the Model interface.
+The database system uses the Record interface to transparently handle all types of structs that get saved in the database. Structs include the Base struct to fulfill most parts of the Record interface.
 
-Boilerplate Code
+Boilerplate Code:
 
-Receiving model, using as struct:
+    type Example struct {
+      record.Base
+      sync.Mutex
 
-  // At some point, declare a pointer to your model.
-  // This is only used to identify the model, so you can reuse it safely for this purpose
-  var cowModel *Cow // only use this as parameter for database.EnsureModel-like functions
-
-  receivedModel := <- models // chan database.Model
-  cow, ok := database.SilentEnsureModel(receivedModel, cowModel).(*Cow)
-  if !ok {
-    panic("received model does not match expected model")
-  }
-
-  // more verbose, in case you need better error handling
-  receivedModel := <- models // chan database.Model
-  genericModel, err := database.EnsureModel(receivedModel, cowModel)
-  if err != nil {
-    panic(err)
-  }
-  cow, ok := genericModel.(*Cow)
-  if !ok {
-    panic("received model does not match expected model")
-  }
-
-Receiving a model, dumping:
-
-  // receivedModel <- chan database.Model
-  bytes, err := database.DumpModel(receivedModel, dsd.JSON) // or other dsd format
-  if err != nil {
-    panic(err)
-  }
-
-Model definition:
-
-  // Cow makes moo.
-  type Cow struct {
-    database.Base
-    // Fields...
-  }
-
-  var cowModel *Cow // only use this as parameter for database.EnsureModel-like functions
-
-  func init() {
-    database.RegisterModel(cowModel, func() database.Model { return new(Cow) })
-  }
-
-  // this all you need, but you might find the following code helpful:
-
-  var cowNamespace = datastore.NewKey("/Cow")
-
-  // Create saves Cow with the provided name in the default namespace.
-  func (m *Cow) Create(name string) error {
-    return m.CreateObject(&cowNamespace, name, m)
-  }
-
-  // CreateInNamespace saves Cow with the provided name in the provided namespace.
-  func (m *Cow) CreateInNamespace(namespace *datastore.Key, name string) error {
-    return m.CreateObject(namespace, name, m)
-  }
-
-  // Save saves Cow.
-  func (m *Cow) Save() error {
-    return m.SaveObject(m)
-  }
-
-  // GetCow fetches Cow with the provided name from the default namespace.
-  func GetCow(name string) (*Cow, error) {
-    return GetCowFromNamespace(&cowNamespace, name)
-  }
-
-  // GetCowFromNamespace fetches Cow with the provided name from the provided namespace.
-  func GetCowFromNamespace(namespace *datastore.Key, name string) (*Cow, error) {
-    object, err := database.GetAndEnsureModel(namespace, name, cowModel)
-    if err != nil {
-      return nil, err
+      Name  string
+      Score int
     }
-    model, ok := object.(*Cow)
-    if !ok {
-      return nil, database.NewMismatchError(object, cowModel)
+
+    var (
+      db = database.NewInterface(nil)
+    )
+
+    // GetExample gets an Example from the database.
+    func GetExample(key string) (*Example, error) {
+      r, err := db.Get(key)
+      if err != nil {
+        return nil, err
+      }
+
+      // unwrap
+      if r.IsWrapped() {
+        // only allocate a new struct, if we need it
+        new := &Example{}
+        err = record.Unwrap(r, new)
+        if err != nil {
+          return nil, err
+        }
+        return new, nil
+      }
+
+      // or adjust type
+      new, ok := r.(*Example)
+      if !ok {
+        return nil, fmt.Errorf("record not of type *Example, but %T", r)
+      }
+      return new, nil
+    }
+
+    func (e *Example) Save() error {
+      return db.Put(e)
+    }
+
+    func (e *Example) SaveAs(key string) error {
+      e.SetKey(key)
+      return db.PutNew(e)
     }
-    return model, nil
-  }
 
 */
 package database
diff --git a/database/hook.go b/database/hook.go
new file mode 100644
index 0000000..a26aa1d
--- /dev/null
+++ b/database/hook.go
@@ -0,0 +1,72 @@
+package database
+
+import (
+	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/record"
+)
+
+// Hook describes a hook
+type Hook interface {
+	UsesPreGet() bool
+	PreGet(dbKey string) error
+
+	UsesPostGet() bool
+	PostGet(r record.Record) (record.Record, error)
+
+	UsesPrePut() bool
+	PrePut(r record.Record) (record.Record, error)
+
+	UsesPostPut() bool
+	PostPut(r record.Record)
+}
+
+// RegisteredHook is a registered database hook.
+type RegisteredHook struct {
+	q    *query.Query
+	hook Hook
+}
+
+// RegisterHook registeres a hook for records matching the given query in the database.
+func RegisterHook(q *query.Query, hook Hook) error {
+	_, err := q.Check()
+	if err != nil {
+		return err
+	}
+
+	c, err := getController(q.DatabaseName())
+	if err != nil {
+		return err
+	}
+
+	c.readLock.Lock()
+	defer c.readLock.Lock()
+	c.writeLock.Lock()
+	defer c.writeLock.Unlock()
+
+	c.hooks = append(c.hooks, &RegisteredHook{
+		q:    q,
+		hook: hook,
+	})
+	return nil
+}
+
+// Cancel unhooks the hook.
+func (h *RegisteredHook) Cancel() error {
+	c, err := getController(h.q.DatabaseName())
+	if err != nil {
+		return err
+	}
+
+	c.readLock.Lock()
+	defer c.readLock.Lock()
+	c.writeLock.Lock()
+	defer c.writeLock.Unlock()
+
+	for key, hook := range c.hooks {
+		if hook.q == h.q {
+			c.hooks = append(c.hooks[:key], c.hooks[key+1:]...)
+			return nil
+		}
+	}
+	return nil
+}
diff --git a/database/hookbase.go b/database/hookbase.go
new file mode 100644
index 0000000..15fc8fe
--- /dev/null
+++ b/database/hookbase.go
@@ -0,0 +1,29 @@
+package database
+
+import (
+	"github.com/Safing/portbase/database/record"
+)
+
+// HookBase implements the Hook interface.
+type HookBase struct {
+}
+
+// PreGet implements the Hook interface.
+func (b *HookBase) PreGet(dbKey string) error {
+	return nil
+}
+
+// PostGet implements the Hook interface.
+func (b *HookBase) PostGet(r record.Record) (record.Record, error) {
+	return r, nil
+}
+
+// PrePut implements the Hook interface.
+func (b *HookBase) PrePut(r record.Record) (record.Record, error) {
+	return r, nil
+}
+
+// PostPut implements the Hook interface.
+func (b *HookBase) PostPut(r record.Record) {
+	return
+}
diff --git a/database/iterator/iterator.go b/database/iterator/iterator.go
index 210731a..7a1dff4 100644
--- a/database/iterator/iterator.go
+++ b/database/iterator/iterator.go
@@ -7,6 +7,7 @@ import (
 // Iterator defines the iterator structure.
 type Iterator struct {
 	Next  chan record.Record
+	Done  chan struct{}
 	Error error
 }
 
@@ -14,5 +15,6 @@ type Iterator struct {
 func New() *Iterator {
 	return &Iterator{
 		Next: make(chan record.Record, 10),
+		Done: make(chan struct{}),
 	}
 }
diff --git a/database/query/query.go b/database/query/query.go
index 0f23023..6741cc7 100644
--- a/database/query/query.go
+++ b/database/query/query.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/Safing/portbase/database/accessor"
 	"github.com/Safing/portbase/database/record"
 )
 
@@ -74,8 +73,6 @@ func (q *Query) Check() (*Query, error) {
 		if err != nil {
 			return nil, err
 		}
-	} else {
-		q.where = &noCond{}
 	}
 
 	q.checked = true
@@ -96,8 +93,28 @@ func (q *Query) IsChecked() bool {
 	return q.checked
 }
 
+// MatchesKey checks whether the query matches the supplied database key (key without database prefix).
+func (q *Query) MatchesKey(dbKey string) bool {
+	if !strings.HasPrefix(dbKey, q.dbKeyPrefix) {
+		return false
+	}
+	return true
+}
+
 // Matches checks whether the query matches the supplied data object.
-func (q *Query) Matches(acc accessor.Accessor) bool {
+func (q *Query) Matches(r record.Record) bool {
+	if !strings.HasPrefix(r.DatabaseKey(), q.dbKeyPrefix) {
+		return false
+	}
+
+	if q.where == nil {
+		return false
+	}
+
+	acc := r.GetAccessor(r)
+	if acc == nil {
+		return false
+	}
 	return q.where.complies(acc)
 }
 
diff --git a/database/record/base.go b/database/record/base.go
index 44d9b54..0c7597f 100644
--- a/database/record/base.go
+++ b/database/record/base.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/Safing/portbase/container"
+	"github.com/Safing/portbase/database/accessor"
 	"github.com/Safing/portbase/formats/dsd"
 )
 
@@ -98,3 +99,8 @@ func (b *Base) MarshalRecord(self Record) ([]byte, error) {
 func (b *Base) IsWrapped() bool {
 	return false
 }
+
+// GetAccessor returns an accessor for this record, if available.
+func (b *Base) GetAccessor(self Record) accessor.Accessor {
+	return accessor.NewStructAccessor(self)
+}
diff --git a/database/record/record.go b/database/record/record.go
index a09de4b..3895292 100644
--- a/database/record/record.go
+++ b/database/record/record.go
@@ -1,5 +1,9 @@
 package record
 
+import (
+	"github.com/Safing/portbase/database/accessor"
+)
+
 // Record provides an interface for uniformally handling database records.
 type Record interface {
 	Key() string          // test:config
@@ -11,8 +15,9 @@ type Record interface {
 	Meta() *Meta
 	SetMeta(meta *Meta)
 
-	Marshal(r Record, format uint8) ([]byte, error)
-	MarshalRecord(r Record) ([]byte, error)
+	Marshal(self Record, format uint8) ([]byte, error)
+	MarshalRecord(self Record) ([]byte, error)
+	GetAccessor(self Record) accessor.Accessor
 
 	Lock()
 	Unlock()
diff --git a/database/record/wrapper.go b/database/record/wrapper.go
index 6e7ef69..63bdb66 100644
--- a/database/record/wrapper.go
+++ b/database/record/wrapper.go
@@ -6,19 +6,26 @@ import (
 	"sync"
 
 	"github.com/Safing/portbase/container"
+	"github.com/Safing/portbase/database/accessor"
 	"github.com/Safing/portbase/formats/dsd"
 	"github.com/Safing/portbase/formats/varint"
 )
 
+// Wrapper wraps raw data and implements the Record interface.
 type Wrapper struct {
 	Base
+	sync.Mutex
+
 	Format uint8
 	Data   []byte
-	lock   sync.Mutex
 }
 
+// NewRawWrapper returns a record wrapper for the given data, including metadata. This is normally only used by storage backends when loading records.
 func NewRawWrapper(database, key string, data []byte) (*Wrapper, error) {
 	version, offset, err := varint.Unpack8(data)
+	if err != nil {
+		return nil, err
+	}
 	if version != 1 {
 		return nil, fmt.Errorf("incompatible record version: %d", version)
 	}
@@ -46,13 +53,13 @@ func NewRawWrapper(database, key string, data []byte) (*Wrapper, error) {
 			key,
 			newMeta,
 		},
+		sync.Mutex{},
 		format,
 		data[offset:],
-		sync.Mutex{},
 	}, nil
 }
 
-// NewWrapper returns a new model wrapper for the given data.
+// NewWrapper returns a new record wrapper for the given data.
 func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
 	format, _, err := varint.Unpack8(data)
 	if err != nil {
@@ -67,9 +74,9 @@ func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
 			dbKey:  dbKey,
 			meta:   meta,
 		},
+		sync.Mutex{},
 		format,
 		data,
-		sync.Mutex{},
 	}, nil
 }
 
@@ -117,34 +124,44 @@ func (w *Wrapper) MarshalRecord(r Record) ([]byte, error) {
 	return c.CompileData(), nil
 }
 
-// Lock locks the record.
-func (w *Wrapper) Lock() {
-	w.lock.Lock()
-}
-
-// Unlock unlocks the record.
-func (w *Wrapper) Unlock() {
-	w.lock.Unlock()
-}
+// // Lock locks the record.
+// func (w *Wrapper) Lock() {
+// 	w.lock.Lock()
+// }
+//
+// // Unlock unlocks the record.
+// func (w *Wrapper) Unlock() {
+// 	w.lock.Unlock()
+// }
 
 // IsWrapped returns whether the record is a Wrapper.
 func (w *Wrapper) IsWrapped() bool {
 	return true
 }
 
-func Unwrap(wrapped, new Record) (Record, error) {
+// Unwrap unwraps data into a record.
+func Unwrap(wrapped, new Record) error {
 	wrapper, ok := wrapped.(*Wrapper)
 	if !ok {
-		return nil, fmt.Errorf("cannot unwrap %T", wrapped)
+		return fmt.Errorf("cannot unwrap %T", wrapped)
 	}
 
 	_, err := dsd.Load(wrapper.Data, new)
 	if err != nil {
-		return nil, fmt.Errorf("database: failed to unwrap %T: %s", new, err)
+		return fmt.Errorf("failed to unwrap %T: %s", new, err)
 	}
 
 	new.SetKey(wrapped.Key())
 	new.SetMeta(wrapped.Meta())
 
-	return new, nil
+	return nil
+}
+
+// GetAccessor returns an accessor for this record, if available.
+func (w *Wrapper) GetAccessor(self Record) accessor.Accessor {
+	if len(w.Data) > 1 && w.Data[0] == JSON {
+		jsonData := w.Data[1:]
+		return accessor.NewJSONBytesAccessor(&jsonData)
+	}
+	return nil
 }
diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
index f685d90..96f63c5 100644
--- a/database/storage/badger/badger.go
+++ b/database/storage/badger/badger.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/dgraph-io/badger"
 
-	"github.com/Safing/portbase/database/accessor"
 	"github.com/Safing/portbase/database/iterator"
 	"github.com/Safing/portbase/database/query"
 	"github.com/Safing/portbase/database/record"
@@ -139,29 +138,24 @@ func (b *Badger) queryExecutor(queryIter *iterator.Iterator, q *query.Query, loc
 				continue
 			}
 
-			if len(r.Data) > 1 {
-				jsonData := r.Data[1:]
-				acc := accessor.NewJSONBytesAccessor(&jsonData)
-				if q.Matches(acc) {
-
-					copiedData, err := item.ValueCopy(nil)
-					if err != nil {
-						return err
-					}
-					new, err := record.NewRawWrapper(b.name, string(item.Key()), copiedData)
-					if err != nil {
-						return err
-					}
+			acc := r.GetAccessor(r)
+			if acc != nil && q.Matches(acc) {
+				copiedData, err := item.ValueCopy(nil)
+				if err != nil {
+					return err
+				}
+				new, err := record.NewRawWrapper(b.name, string(item.Key()), copiedData)
+				if err != nil {
+					return err
+				}
+				select {
+				case queryIter.Next <- new:
+				default:
 					select {
 					case queryIter.Next <- new:
-					default:
-						select {
-						case queryIter.Next <- new:
-						case <-time.After(1 * time.Minute):
-							return errors.New("query timeout")
-						}
+					case <-time.After(1 * time.Minute):
+						return errors.New("query timeout")
 					}
-
 				}
 			}
 
@@ -173,6 +167,7 @@ func (b *Badger) queryExecutor(queryIter *iterator.Iterator, q *query.Query, loc
 		queryIter.Error = err
 	}
 	close(queryIter.Next)
+	close(queryIter.Done)
 }
 
 // ReadOnly returns whether the database is read only.
diff --git a/database/storage/badger/badger_test.go b/database/storage/badger/badger_test.go
index ee22fb2..e858769 100644
--- a/database/storage/badger/badger_test.go
+++ b/database/storage/badger/badger_test.go
@@ -78,7 +78,7 @@ func TestBadger(t *testing.T) {
 	}
 
 	a1 := &TestRecord{}
-	_, err = record.Unwrap(r1, a1)
+	err = record.Unwrap(r1, a1)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/database/subscription.go b/database/subscription.go
new file mode 100644
index 0000000..592e2c7
--- /dev/null
+++ b/database/subscription.go
@@ -0,0 +1,59 @@
+package database
+
+import (
+	"github.com/Safing/portbase/database/query"
+	"github.com/Safing/portbase/database/record"
+)
+
+// Subscription is a database subscription for updates.
+type Subscription struct {
+	q    *query.Query
+	Feed chan record.Record
+	Err  error
+}
+
+// Subscribe subscribes to updates matching the given query.
+func Subscribe(q *query.Query) (*Subscription, error) {
+	_, err := q.Check()
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := getController(q.DatabaseName())
+	if err != nil {
+		return nil, err
+	}
+
+	c.readLock.Lock()
+	defer c.readLock.Lock()
+	c.writeLock.Lock()
+	defer c.writeLock.Unlock()
+
+	sub := &Subscription{
+		q:    q,
+		Feed: make(chan record.Record, 100),
+	}
+	c.subscriptions = append(c.subscriptions, sub)
+	return sub, nil
+}
+
+// Cancel cancels the subscription.
+func (s *Subscription) Cancel() error {
+	c, err := getController(s.q.DatabaseName())
+	if err != nil {
+		return err
+	}
+
+	c.readLock.Lock()
+	defer c.readLock.Lock()
+	c.writeLock.Lock()
+	defer c.writeLock.Unlock()
+
+	for key, sub := range c.subscriptions {
+		if sub.q == s.q {
+			c.subscriptions = append(c.subscriptions[:key], c.subscriptions[key+1:]...)
+			return nil
+		}
+	}
+	return nil
+}

From ac13b73d653ecd537e217aa901c7817bc648d1a4 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 14 Sep 2018 11:59:30 +0200
Subject: [PATCH 20/20] Complete database MVP

---
 database/controller.go                 |  29 +--
 database/database_test.go              |  24 ++-
 database/hook.go                       |  28 ++-
 database/hookbase.go                   |  22 +-
 database/query/condition-no.go         |  20 --
 database/query/query.go                |  40 ++--
 database/query/query_test.go           |  67 +++---
 database/storage/badger/badger.go      |   5 +-
 database/storage/badger/badger_test.go |  26 ++-
 database/subscription.go               |   4 +-
 database/subscriptions.go              | 272 -------------------------
 database/subscriptions_test.go         | 103 ----------
 12 files changed, 160 insertions(+), 480 deletions(-)
 delete mode 100644 database/query/condition-no.go
 delete mode 100644 database/subscriptions.go
 delete mode 100644 database/subscriptions_test.go

diff --git a/database/controller.go b/database/controller.go
index d2f279c..438072f 100644
--- a/database/controller.go
+++ b/database/controller.go
@@ -54,8 +54,8 @@ func (c *Controller) Get(key string) (record.Record, error) {
 
 	// process hooks
 	for _, hook := range c.hooks {
-		if hook.q.MatchesKey(key) {
-			err := hook.hook.PreGet(key)
+		if hook.h.UsesPreGet() && hook.q.MatchesKey(key) {
+			err := hook.h.PreGet(key)
 			if err != nil {
 				return nil, err
 			}
@@ -76,8 +76,8 @@ func (c *Controller) Get(key string) (record.Record, error) {
 
 	// process hooks
 	for _, hook := range c.hooks {
-		if hook.q.Matches(r) {
-			r, err = hook.hook.PostGet(r)
+		if hook.h.UsesPostGet() && hook.q.Matches(r) {
+			r, err = hook.h.PostGet(r)
 			if err != nil {
 				return nil, err
 			}
@@ -106,13 +106,13 @@ func (c *Controller) Put(r record.Record) (err error) {
 
 	// process hooks
 	for _, hook := range c.hooks {
-		if hook.q.Matches(r) {
-		r, err = hook.hook.PrePut(r)
-		if err != nil {
-			return err
+		if hook.h.UsesPrePut() && hook.q.Matches(r) {
+			r, err = hook.h.PrePut(r)
+			if err != nil {
+				return err
+			}
 		}
 	}
-	}
 
 	if r.Meta() == nil {
 		r.SetMeta(&record.Meta{})
@@ -127,10 +127,13 @@ func (c *Controller) Put(r record.Record) (err error) {
 		return err
 	}
 
-	// process hooks
-	for _, hook := range c.hooks {
-		if hook.q.Matches(r) {
-			hook.hook.PostPut(r)
+	// process subscriptions
+	for _, sub := range c.subscriptions {
+		if sub.q.Matches(r) {
+			select {
+			case sub.Feed <- r:
+			default:
+			}
 		}
 	}
 
diff --git a/database/database_test.go b/database/database_test.go
index d38f575..4cc94c3 100644
--- a/database/database_test.go
+++ b/database/database_test.go
@@ -30,6 +30,19 @@ func testDatabase(t *testing.T, storageType string) {
 		t.Fatal(err)
 	}
 
+	// hook
+	hook, err := RegisterHook(q.New(dbName).MustBeValid(), &HookBase{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// sub
+	sub, err := Subscribe(q.New(dbName).MustBeValid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// interface
 	db := NewInterface(nil)
 
 	A := NewExample(makeKey(dbName, "A"), "Herbert", 411)
@@ -92,6 +105,15 @@ func testDatabase(t *testing.T, storageType string) {
 		t.Fatal("expected two records")
 	}
 
+	err = hook.Cancel()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = sub.Cancel()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 }
 
 func TestDatabaseSystem(t *testing.T) {
@@ -99,7 +121,7 @@ func TestDatabaseSystem(t *testing.T) {
 	// panic after 10 seconds, to check for locks
 	go func() {
 		time.Sleep(10 * time.Second)
-		fmt.Println("===== TAKING TOO LONG FOR SHUTDOWN - PRINTING STACK TRACES =====")
+		fmt.Println("===== TAKING TOO LONG - PRINTING STACK TRACES =====")
 		pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
 		os.Exit(1)
 	}()
diff --git a/database/hook.go b/database/hook.go
index a26aa1d..1edac94 100644
--- a/database/hook.go
+++ b/database/hook.go
@@ -15,39 +15,37 @@ type Hook interface {
 
 	UsesPrePut() bool
 	PrePut(r record.Record) (record.Record, error)
-
-	UsesPostPut() bool
-	PostPut(r record.Record)
 }
 
 // RegisteredHook is a registered database hook.
 type RegisteredHook struct {
-	q    *query.Query
-	hook Hook
+	q *query.Query
+	h Hook
 }
 
 // RegisterHook registeres a hook for records matching the given query in the database.
-func RegisterHook(q *query.Query, hook Hook) error {
+func RegisterHook(q *query.Query, hook Hook) (*RegisteredHook, error) {
 	_, err := q.Check()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	c, err := getController(q.DatabaseName())
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	c.readLock.Lock()
-	defer c.readLock.Lock()
+	defer c.readLock.Unlock()
 	c.writeLock.Lock()
 	defer c.writeLock.Unlock()
 
-	c.hooks = append(c.hooks, &RegisteredHook{
-		q:    q,
-		hook: hook,
-	})
-	return nil
+	rh := &RegisteredHook{
+		q: q,
+		h: hook,
+	}
+	c.hooks = append(c.hooks, rh)
+	return rh, nil
 }
 
 // Cancel unhooks the hook.
@@ -58,7 +56,7 @@ func (h *RegisteredHook) Cancel() error {
 	}
 
 	c.readLock.Lock()
-	defer c.readLock.Lock()
+	defer c.readLock.Unlock()
 	c.writeLock.Lock()
 	defer c.writeLock.Unlock()
 
diff --git a/database/hookbase.go b/database/hookbase.go
index 15fc8fe..fd42748 100644
--- a/database/hookbase.go
+++ b/database/hookbase.go
@@ -4,10 +4,25 @@ import (
 	"github.com/Safing/portbase/database/record"
 )
 
-// HookBase implements the Hook interface.
+// HookBase implements the Hook interface and provides dummy functions to reduce boilerplate.
 type HookBase struct {
 }
 
+// UsesPreGet implements the Hook interface and returns false.
+func (b *HookBase) UsesPreGet() bool {
+	return false
+}
+
+// UsesPostGet implements the Hook interface and returns false.
+func (b *HookBase) UsesPostGet() bool {
+	return false
+}
+
+// UsesPrePut implements the Hook interface and returns false.
+func (b *HookBase) UsesPrePut() bool {
+	return false
+}
+
 // PreGet implements the Hook interface.
 func (b *HookBase) PreGet(dbKey string) error {
 	return nil
@@ -22,8 +37,3 @@ func (b *HookBase) PostGet(r record.Record) (record.Record, error) {
 func (b *HookBase) PrePut(r record.Record) (record.Record, error) {
 	return r, nil
 }
-
-// PostPut implements the Hook interface.
-func (b *HookBase) PostPut(r record.Record) {
-	return
-}
diff --git a/database/query/condition-no.go b/database/query/condition-no.go
deleted file mode 100644
index 8709474..0000000
--- a/database/query/condition-no.go
+++ /dev/null
@@ -1,20 +0,0 @@
-package query
-
-import (
-	"github.com/Safing/portbase/database/accessor"
-)
-
-type noCond struct {
-}
-
-func (c *noCond) complies(acc accessor.Accessor) bool {
-	return true
-}
-
-func (c *noCond) check() (err error) {
-	return nil
-}
-
-func (c *noCond) string() string {
-	return ""
-}
diff --git a/database/query/query.go b/database/query/query.go
index 6741cc7..d9ecb7f 100644
--- a/database/query/query.go
+++ b/database/query/query.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/Safing/portbase/database/accessor"
 	"github.com/Safing/portbase/database/record"
 )
 
@@ -101,14 +102,10 @@ func (q *Query) MatchesKey(dbKey string) bool {
 	return true
 }
 
-// Matches checks whether the query matches the supplied data object.
-func (q *Query) Matches(r record.Record) bool {
-	if !strings.HasPrefix(r.DatabaseKey(), q.dbKeyPrefix) {
-		return false
-	}
-
+// MatchesRecord checks whether the query matches the supplied database record (value only).
+func (q *Query) MatchesRecord(r record.Record) bool {
 	if q.where == nil {
-		return false
+		return true
 	}
 
 	acc := r.GetAccessor(r)
@@ -118,14 +115,33 @@ func (q *Query) Matches(r record.Record) bool {
 	return q.where.complies(acc)
 }
 
+// MatchesAccessor checks whether the query matches the supplied accessor (value only).
+func (q *Query) MatchesAccessor(acc accessor.Accessor) bool {
+	if q.where == nil {
+		return true
+	}
+	return q.where.complies(acc)
+}
+
+// Matches checks whether the query matches the supplied database record.
+func (q *Query) Matches(r record.Record) bool {
+	if q.MatchesKey(r.DatabaseKey()) {
+		return true
+	}
+	return q.MatchesRecord(r)
+}
+
 // Print returns the string representation of the query.
 func (q *Query) Print() string {
-	where := q.where.string()
-	if where != "" {
-		if strings.HasPrefix(where, "(") {
-			where = where[1 : len(where)-1]
+	var where string
+	if q.where != nil {
+		where = q.where.string()
+		if where != "" {
+			if strings.HasPrefix(where, "(") {
+				where = where[1 : len(where)-1]
+			}
+			where = fmt.Sprintf(" where %s", where)
 		}
-		where = fmt.Sprintf(" where %s", where)
 	}
 
 	var orderBy string
diff --git a/database/query/query_test.go b/database/query/query_test.go
index 6ac49f8..4645a61 100644
--- a/database/query/query_test.go
+++ b/database/query/query_test.go
@@ -3,7 +3,7 @@ package query
 import (
 	"testing"
 
-	"github.com/Safing/portbase/database/accessor"
+	"github.com/Safing/portbase/database/record"
 )
 
 var (
@@ -46,12 +46,12 @@ var (
 }`
 )
 
-func testQuery(t *testing.T, acc accessor.Accessor, shouldMatch bool, condition Condition) {
+func testQuery(t *testing.T, r record.Record, shouldMatch bool, condition Condition) {
 	q := New("test:").Where(condition).MustBeValid()
 
-	// fmt.Printf("%s\n", q.String())
+	// fmt.Printf("%s\n", q.Print())
 
-	matched := q.Matches(acc)
+	matched := q.Matches(r)
 	switch {
 	case !matched && shouldMatch:
 		t.Errorf("should match: %s", q.Print())
@@ -65,36 +65,39 @@ func TestQuery(t *testing.T) {
 	// if !gjson.Valid(testJSON) {
 	// 	t.Fatal("test json is invalid")
 	// }
-	f := accessor.NewJSONAccessor(&testJSON)
+	r, err := record.NewWrapper("", nil, append([]byte("J"), []byte(testJSON)...))
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	testQuery(t, f, true, Where("age", Equals, 100))
-	testQuery(t, f, true, Where("age", GreaterThan, uint8(99)))
-	testQuery(t, f, true, Where("age", GreaterThanOrEqual, 99))
-	testQuery(t, f, true, Where("age", GreaterThanOrEqual, 100))
-	testQuery(t, f, true, Where("age", LessThan, 101))
-	testQuery(t, f, true, Where("age", LessThanOrEqual, "101"))
-	testQuery(t, f, true, Where("age", LessThanOrEqual, 100))
+	testQuery(t, r, true, Where("age", Equals, 100))
+	testQuery(t, r, true, Where("age", GreaterThan, uint8(99)))
+	testQuery(t, r, true, Where("age", GreaterThanOrEqual, 99))
+	testQuery(t, r, true, Where("age", GreaterThanOrEqual, 100))
+	testQuery(t, r, true, Where("age", LessThan, 101))
+	testQuery(t, r, true, Where("age", LessThanOrEqual, "101"))
+	testQuery(t, r, true, Where("age", LessThanOrEqual, 100))
 
-	testQuery(t, f, true, Where("temperature", FloatEquals, 120.413))
-	testQuery(t, f, true, Where("temperature", FloatGreaterThan, 120))
-	testQuery(t, f, true, Where("temperature", FloatGreaterThanOrEqual, 120))
-	testQuery(t, f, true, Where("temperature", FloatGreaterThanOrEqual, 120.413))
-	testQuery(t, f, true, Where("temperature", FloatLessThan, 121))
-	testQuery(t, f, true, Where("temperature", FloatLessThanOrEqual, "121"))
-	testQuery(t, f, true, Where("temperature", FloatLessThanOrEqual, "120.413"))
+	testQuery(t, r, true, Where("temperature", FloatEquals, 120.413))
+	testQuery(t, r, true, Where("temperature", FloatGreaterThan, 120))
+	testQuery(t, r, true, Where("temperature", FloatGreaterThanOrEqual, 120))
+	testQuery(t, r, true, Where("temperature", FloatGreaterThanOrEqual, 120.413))
+	testQuery(t, r, true, Where("temperature", FloatLessThan, 121))
+	testQuery(t, r, true, Where("temperature", FloatLessThanOrEqual, "121"))
+	testQuery(t, r, true, Where("temperature", FloatLessThanOrEqual, "120.413"))
 
-	testQuery(t, f, true, Where("lastly.yay", SameAs, "final"))
-	testQuery(t, f, true, Where("lastly.yay", Contains, "ina"))
-	testQuery(t, f, true, Where("lastly.yay", StartsWith, "fin"))
-	testQuery(t, f, true, Where("lastly.yay", EndsWith, "nal"))
-	testQuery(t, f, true, Where("lastly.yay", In, "draft,final"))
-	testQuery(t, f, true, Where("lastly.yay", In, "final,draft"))
+	testQuery(t, r, true, Where("lastly.yay", SameAs, "final"))
+	testQuery(t, r, true, Where("lastly.yay", Contains, "ina"))
+	testQuery(t, r, true, Where("lastly.yay", StartsWith, "fin"))
+	testQuery(t, r, true, Where("lastly.yay", EndsWith, "nal"))
+	testQuery(t, r, true, Where("lastly.yay", In, "draft,final"))
+	testQuery(t, r, true, Where("lastly.yay", In, "final,draft"))
 
-	testQuery(t, f, true, Where("happy", Is, true))
-	testQuery(t, f, true, Where("happy", Is, "true"))
-	testQuery(t, f, true, Where("happy", Is, "t"))
-	testQuery(t, f, true, Not(Where("happy", Is, "0")))
-	testQuery(t, f, true, And(
+	testQuery(t, r, true, Where("happy", Is, true))
+	testQuery(t, r, true, Where("happy", Is, "true"))
+	testQuery(t, r, true, Where("happy", Is, "t"))
+	testQuery(t, r, true, Not(Where("happy", Is, "0")))
+	testQuery(t, r, true, And(
 		Where("happy", Is, "1"),
 		Not(Or(
 			Where("happy", Is, false),
@@ -102,8 +105,8 @@ func TestQuery(t *testing.T) {
 		)),
 	))
 
-	testQuery(t, f, true, Where("happy", Exists, nil))
+	testQuery(t, r, true, Where("happy", Exists, nil))
 
-	testQuery(t, f, true, Where("created", Matches, "^2014-[0-9]{2}-[0-9]{2}T"))
+	testQuery(t, r, true, Where("created", Matches, "^2014-[0-9]{2}-[0-9]{2}T"))
 
 }
diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go
index 96f63c5..0ff6abe 100644
--- a/database/storage/badger/badger.go
+++ b/database/storage/badger/badger.go
@@ -138,13 +138,12 @@ func (b *Badger) queryExecutor(queryIter *iterator.Iterator, q *query.Query, loc
 				continue
 			}
 
-			acc := r.GetAccessor(r)
-			if acc != nil && q.Matches(acc) {
+			if q.MatchesRecord(r) {
 				copiedData, err := item.ValueCopy(nil)
 				if err != nil {
 					return err
 				}
-				new, err := record.NewRawWrapper(b.name, string(item.Key()), copiedData)
+				new, err := record.NewRawWrapper(b.name, r.DatabaseKey(), copiedData)
 				if err != nil {
 					return err
 				}
diff --git a/database/storage/badger/badger_test.go b/database/storage/badger/badger_test.go
index e858769..f89ee90 100644
--- a/database/storage/badger/badger_test.go
+++ b/database/storage/badger/badger_test.go
@@ -7,6 +7,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/Safing/portbase/database/query"
 	"github.com/Safing/portbase/database/record"
 )
 
@@ -42,6 +43,7 @@ func TestBadger(t *testing.T) {
 	}
 	defer os.RemoveAll(testDir) // clean up
 
+	// start
 	db, err := NewBadger("test", testDir)
 	if err != nil {
 		t.Fatal(err)
@@ -67,11 +69,13 @@ func TestBadger(t *testing.T) {
 	a.Meta().Update()
 	a.SetKey("test:A")
 
+	// put record
 	err = db.Put(a)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	// get and compare
 	r1, err := db.Get("A")
 	if err != nil {
 		t.Fatal(err)
@@ -87,26 +91,46 @@ func TestBadger(t *testing.T) {
 		t.Fatalf("mismatch, got %v", a1)
 	}
 
+	// test query
+	q := query.New("").MustBeValid()
+	it, err := db.Query(q, true, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cnt := 0
+	for _ = range it.Next {
+		cnt++
+	}
+	if it.Error != nil {
+		t.Fatal(err)
+	}
+	if cnt != 1 {
+		t.Fatalf("unexpected query result count: %d", cnt)
+	}
+
+	// delete
 	err = db.Delete("A")
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	// check if its gone
 	_, err = db.Get("A")
 	if err == nil {
 		t.Fatal("should fail")
 	}
 
+	// maintenance
 	err = db.Maintain()
 	if err != nil {
 		t.Fatal(err)
 	}
-
 	err = db.MaintainThorough()
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	// shutdown
 	err = db.Shutdown()
 	if err != nil {
 		t.Fatal(err)
diff --git a/database/subscription.go b/database/subscription.go
index 592e2c7..d95ac94 100644
--- a/database/subscription.go
+++ b/database/subscription.go
@@ -25,7 +25,7 @@ func Subscribe(q *query.Query) (*Subscription, error) {
 	}
 
 	c.readLock.Lock()
-	defer c.readLock.Lock()
+	defer c.readLock.Unlock()
 	c.writeLock.Lock()
 	defer c.writeLock.Unlock()
 
@@ -45,7 +45,7 @@ func (s *Subscription) Cancel() error {
 	}
 
 	c.readLock.Lock()
-	defer c.readLock.Lock()
+	defer c.readLock.Unlock()
 	c.writeLock.Lock()
 	defer c.writeLock.Unlock()
 
diff --git a/database/subscriptions.go b/database/subscriptions.go
deleted file mode 100644
index 460713e..0000000
--- a/database/subscriptions.go
+++ /dev/null
@@ -1,272 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-// import (
-// 	"fmt"
-// 	"strings"
-// 	"sync"
-//
-// 	"github.com/Safing/portbase/database/record"
-// 	"github.com/Safing/portbase/modules"
-// 	"github.com/Safing/portbase/taskmanager"
-//
-// 	"github.com/tevino/abool"
-// )
-//
-// var subscriptionModule *modules.Module
-// var subscriptions []*Subscription
-// var subLock sync.Mutex
-//
-// var databaseUpdate chan Model
-// var databaseCreate chan Model
-// var databaseDelete chan string
-//
-// var workIsWaiting chan *struct{}
-// var workIsWaitingFlag *abool.AtomicBool
-// var forceProcessing chan *struct{}
-//
-// type Subscription struct {
-// 	typeAndLocation map[string]bool
-// 	exactObject     map[string]bool
-// 	children        map[string]uint8
-// 	Created         chan record.Record
-// 	Updated         chan record.Record
-// 	Deleted         chan string
-// }
-//
-// func NewSubscription() *Subscription {
-// 	subLock.Lock()
-// 	defer subLock.Unlock()
-// 	sub := &Subscription{
-// 		typeAndLocation: make(map[string]bool),
-// 		exactObject:     make(map[string]bool),
-// 		children:        make(map[string]uint8),
-// 		Created:         make(chan record.Record, 128),
-// 		Updated:         make(chan record.Record, 128),
-// 		Deleted:         make(chan string, 128),
-// 	}
-// 	subscriptions = append(subscriptions, sub)
-// 	return sub
-// }
-//
-// func (sub *Subscription) Subscribe(subKey string) {
-// 	subLock.Lock()
-// 	defer subLock.Unlock()
-//
-// 	namespaces := strings.Split(subKey, "/")[1:]
-// 	lastSpace := ""
-// 	if len(namespaces) != 0 {
-// 		lastSpace = namespaces[len(namespaces)-1]
-// 	}
-//
-// 	switch {
-// 	case lastSpace == "":
-// 		// save key without leading "/"
-// 		// save with depth 255 to get all
-// 		sub.children[strings.Trim(subKey, "/")] = 0xFF
-// 	case strings.HasPrefix(lastSpace, "*"):
-// 		// save key without leading or trailing "/" or "*"
-// 		// save full wanted depth - this makes comparison easier
-// 		sub.children[strings.Trim(subKey, "/*")] = uint8(len(lastSpace) + len(namespaces) - 1)
-// 	case strings.Contains(lastSpace, ":"):
-// 		sub.exactObject[subKey] = true
-// 	default:
-// 		sub.typeAndLocation[subKey] = true
-// 	}
-// }
-//
-// func (sub *Subscription) Unsubscribe(subKey string) {
-// 	subLock.Lock()
-// 	defer subLock.Unlock()
-//
-// 	namespaces := strings.Split(subKey, "/")[1:]
-// 	lastSpace := ""
-// 	if len(namespaces) != 0 {
-// 		lastSpace = namespaces[len(namespaces)-1]
-// 	}
-//
-// 	switch {
-// 	case lastSpace == "":
-// 		delete(sub.children, strings.Trim(subKey, "/"))
-// 	case strings.HasPrefix(lastSpace, "*"):
-// 		delete(sub.children, strings.Trim(subKey, "/*"))
-// 	case strings.Contains(lastSpace, ":"):
-// 		delete(sub.exactObject, subKey)
-// 	default:
-// 		delete(sub.typeAndLocation, subKey)
-// 	}
-// }
-//
-// func (sub *Subscription) Destroy() {
-// 	subLock.Lock()
-// 	defer subLock.Unlock()
-//
-// 	for k, v := range subscriptions {
-// 		if v.Created == sub.Created {
-// 			defer func() {
-// 				subscriptions = append(subscriptions[:k], subscriptions[k+1:]...)
-// 			}()
-// 			close(sub.Created)
-// 			close(sub.Updated)
-// 			close(sub.Deleted)
-// 			return
-// 		}
-// 	}
-// }
-//
-// func (sub *Subscription) Subscriptions() *[]string {
-// 	subStrings := make([]string, 0)
-// 	for subString := range sub.exactObject {
-// 		subStrings = append(subStrings, subString)
-// 	}
-// 	for subString := range sub.typeAndLocation {
-// 		subStrings = append(subStrings, subString)
-// 	}
-// 	for subString, depth := range sub.children {
-// 		if depth == 0xFF {
-// 			subStrings = append(subStrings, fmt.Sprintf("/%s/", subString))
-// 		} else {
-// 			subStrings = append(subStrings, fmt.Sprintf("/%s/%s", subString, strings.Repeat("*", int(depth)-len(strings.Split(subString, "/")))))
-// 		}
-// 	}
-// 	return &subStrings
-// }
-//
-// func (sub *Subscription) String() string {
-// 	return fmt.Sprintf("<Subscription [%s]>", strings.Join(*sub.Subscriptions(), " "))
-// }
-//
-// func (sub *Subscription) send(key string, rec record.Record, created bool) {
-// 	if rec == nil {
-// 		sub.Deleted <- key
-// 	} else if created {
-// 		sub.Created <- rec
-// 	} else {
-// 		sub.Updated <- rec
-// 	}
-// }
-//
-// func process(key string, rec record.Record, created bool) {
-// 	subLock.Lock()
-// 	defer subLock.Unlock()
-//
-// 	stringRep := key.String()
-// 	// "/Comedy/MontyPython/Actor:JohnCleese"
-// 	typeAndLocation := key.Path().String()
-// 	// "/Comedy/MontyPython/Actor"
-// 	namespaces := key.Namespaces()
-// 	// ["Comedy", "MontyPython", "Actor:JohnCleese"]
-// 	depth := uint8(len(namespaces))
-// 	// 3
-//
-// subscriptionLoop:
-// 	for _, sub := range subscriptions {
-// 		if _, ok := sub.exactObject[stringRep]; ok {
-// 			sub.send(key, rec, created)
-// 			continue subscriptionLoop
-// 		}
-// 		if _, ok := sub.typeAndLocation[typeAndLocation]; ok {
-// 			sub.send(key, rec, created)
-// 			continue subscriptionLoop
-// 		}
-// 		for i := 0; i < len(namespaces); i++ {
-// 			if subscribedDepth, ok := sub.children[strings.Join(namespaces[:i], "/")]; ok {
-// 				if subscribedDepth >= depth {
-// 					sub.send(key, rec, created)
-// 					continue subscriptionLoop
-// 				}
-// 			}
-// 		}
-// 	}
-//
-// }
-//
-// func init() {
-// 	subscriptionModule = modules.Register("Database:Subscriptions", 128)
-// 	subscriptions = make([]*Subscription, 0)
-// 	subLock = sync.Mutex{}
-//
-// 	databaseUpdate = make(chan Model, 32)
-// 	databaseCreate = make(chan Model, 32)
-// 	databaseDelete = make(chan string, 32)
-//
-// 	workIsWaiting = make(chan *struct{}, 0)
-// 	workIsWaitingFlag = abool.NewBool(false)
-// 	forceProcessing = make(chan *struct{}, 0)
-//
-// 	go run()
-// }
-//
-// func run() {
-// 	for {
-// 		select {
-// 		case <-subscriptionModule.Stop:
-// 			subscriptionModule.StopComplete()
-// 			return
-// 		case <-workIsWaiting:
-// 			work()
-// 		}
-// 	}
-// }
-//
-// func work() {
-// 	defer workIsWaitingFlag.UnSet()
-//
-// 	// wait
-// 	select {
-// 	case <-taskmanager.StartMediumPriorityMicroTask():
-// 		defer taskmanager.EndMicroTask()
-// 	case <-forceProcessing:
-// 	}
-//
-// 	// work
-// 	for {
-// 		select {
-// 		case rec := <-databaseCreate:
-// 			process(rec.GetKey(), rec, true)
-// 		case rec := <-databaseUpdate:
-// 			process(rec.GetKey(), rec, false)
-// 		case key := <-databaseDelete:
-// 			process(key, nil, false)
-// 		default:
-// 			return
-// 		}
-// 	}
-// }
-//
-// func handleCreateSubscriptions(rec record.Record) {
-// 	select {
-// 	case databaseCreate <- rec:
-// 	default:
-// 		forceProcessing <- nil
-// 		databaseCreate <- rec
-// 	}
-// 	if workIsWaitingFlag.SetToIf(false, true) {
-// 		workIsWaiting <- nil
-// 	}
-// }
-//
-// func handleUpdateSubscriptions(rec record.Record) {
-// 	select {
-// 	case databaseUpdate <- rec:
-// 	default:
-// 		forceProcessing <- nil
-// 		databaseUpdate <- rec
-// 	}
-// 	if workIsWaitingFlag.SetToIf(false, true) {
-// 		workIsWaiting <- nil
-// 	}
-// }
-//
-// func handleDeleteSubscriptions(key string) {
-// 	select {
-// 	case databaseDelete <- key:
-// 	default:
-// 		forceProcessing <- nil
-// 		databaseDelete <- key
-// 	}
-// 	if workIsWaitingFlag.SetToIf(false, true) {
-// 		workIsWaiting <- nil
-// 	}
-// }
diff --git a/database/subscriptions_test.go b/database/subscriptions_test.go
deleted file mode 100644
index 5ab06d3..0000000
--- a/database/subscriptions_test.go
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
-
-package database
-
-// import (
-// 	"strconv"
-// 	"strings"
-// 	"sync"
-// 	"testing"
-// )
-//
-// var subTestWg sync.WaitGroup
-//
-// func waitForSubs(t *testing.T, sub *Subscription, highest int) {
-// 	defer subTestWg.Done()
-// 	expecting := 1
-// 	var subbedModel Model
-// forLoop:
-// 	for {
-// 		select {
-// 		case subbedModel = <-sub.Created:
-// 		case subbedModel = <-sub.Updated:
-// 		}
-// 		t.Logf("got model from subscription: %s", subbedModel.GetKey().String())
-// 		if !strings.HasPrefix(subbedModel.GetKey().Name(), "sub") {
-// 			// not a model that we use for testing, other tests might be interfering
-// 			continue forLoop
-// 		}
-// 		number, err := strconv.Atoi(strings.TrimPrefix(subbedModel.GetKey().Name(), "sub"))
-// 		if err != nil || number != expecting {
-// 			t.Errorf("test subscription: got unexpected model %s, expected sub%d", subbedModel.GetKey().String(), expecting)
-// 			continue forLoop
-// 		}
-// 		if number == highest {
-// 			return
-// 		}
-// 		expecting++
-// 	}
-// }
-//
-// func TestSubscriptions(t *testing.T) {
-//
-// 	// create subscription
-// 	sub := NewSubscription()
-//
-// 	// FIRST TEST
-//
-// 	subTestWg.Add(1)
-// 	go waitForSubs(t, sub, 3)
-// 	sub.Subscribe("/Tests/")
-// 	t.Log(sub.String())
-//
-// 	(&(TestingModel{})).CreateInNamespace("", "sub1")
-// 	(&(TestingModel{})).CreateInNamespace("A", "sub2")
-// 	(&(TestingModel{})).CreateInNamespace("A/B/C/D/E", "sub3")
-//
-// 	subTestWg.Wait()
-//
-// 	// SECOND TEST
-//
-// 	subTestWg.Add(1)
-// 	go waitForSubs(t, sub, 3)
-// 	sub.Unsubscribe("/Tests/")
-// 	sub.Subscribe("/Tests/A/****")
-// 	t.Log(sub.String())
-//
-// 	(&(TestingModel{})).CreateInNamespace("", "subX")
-// 	(&(TestingModel{})).CreateInNamespace("A", "sub1")
-// 	(&(TestingModel{})).CreateInNamespace("A/B/C/D", "sub2")
-// 	(&(TestingModel{})).CreateInNamespace("A/B/C/D/E", "subX")
-// 	(&(TestingModel{})).CreateInNamespace("A", "sub3")
-//
-// 	subTestWg.Wait()
-//
-// 	// THIRD TEST
-//
-// 	subTestWg.Add(1)
-// 	go waitForSubs(t, sub, 3)
-// 	sub.Unsubscribe("/Tests/A/****")
-// 	sub.Subscribe("/Tests/TestingModel:sub1")
-// 	sub.Subscribe("/Tests/TestingModel:sub1/TestingModel")
-// 	t.Log(sub.String())
-//
-// 	(&(TestingModel{})).CreateInNamespace("", "sub1")
-// 	(&(TestingModel{})).CreateInNamespace("", "subX")
-// 	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1", "sub2")
-// 	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1/A", "subX")
-// 	(&(TestingModel{})).CreateInNamespace("TestingModel:sub1", "sub3")
-//
-// 	subTestWg.Wait()
-//
-// 	// FINAL STUFF
-//
-// 	model := &TestingModel{}
-// 	model.CreateInNamespace("Invalid", "subX")
-// 	model.Save()
-//
-// 	sub.Destroy()
-//
-// 	// time.Sleep(1 * time.Second)
-// 	// pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
-//
-// }