vikarti.anatra/safing-portbase
1
0
Fork 0
mirror of https://github.com/safing/portbase synced 2025-09-01 18:19:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Finish minimal feature set, start with tests

Browse source
This commit is contained in:
Daniel 2018-09-10 19:01:28 +02:00
parent 3d60431376
commit 06a34f931e
34 changed files with 651 additions and 346 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
database/record/accessor-json-bytes.go → database/accessor/accessor-json-bytes.go
View file

@ -1,6 +1,8 @@
package record
package accessor
import (
"fmt"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
@ -19,6 +21,24 @@ func NewJSONBytesAccessor(json *[]byte) *JSONBytesAccessor {
// Set sets the value identified by key.
func (ja *JSONBytesAccessor) Set(key string, value interface{}) error {
result := gjson.GetBytes(*ja.json, key)
if result.Exists() {
switch value.(type) {
case string:
if result.Type != gjson.String {
return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
}
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
if result.Type != gjson.Number {
return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
}
case bool:
if result.Type != gjson.True && result.Type != gjson.False {
return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
}
}
}
new, err := sjson.SetBytes(*ja.json, key, value)
if err != nil {
return err

22
database/record/accessor-json-string.go → database/accessor/accessor-json-string.go
View file

@ -1,6 +1,8 @@
package record
package accessor
import (
"fmt"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
@ -19,6 +21,24 @@ func NewJSONAccessor(json *string) *JSONAccessor {
// Set sets the value identified by key.
func (ja *JSONAccessor) Set(key string, value interface{}) error {
result := gjson.Get(*ja.json, key)
if result.Exists() {
switch value.(type) {
case string:
if result.Type != gjson.String {
return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
}
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
if result.Type != gjson.Number {
return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
}
case bool:
if result.Type != gjson.True && result.Type != gjson.False {
return fmt.Errorf("tried to set field %s (%s) to a %T value", key, result.Type.String(), value)
}
}
}
new, err := sjson.Set(*ja.json, key, value)
if err != nil {
return err

2
database/record/accessor-struct.go → database/accessor/accessor-struct.go
View file

@ -1,4 +1,4 @@
package record
package accessor
import (
"errors"

2
database/record/accessor.go → database/accessor/accessor.go
View file

@ -1,4 +1,4 @@
package record
package accessor
const (
emptyString = ""

92
database/record/accessor_test.go → database/accessor/accessor_test.go
View file

@ -1,4 +1,4 @@
package record
package accessor
import (
"encoding/json"
@ -95,6 +95,16 @@ func testGetBool(t *testing.T, acc Accessor, key string, shouldSucceed bool, exp
}
}
func testExists(t *testing.T, acc Accessor, key string, shouldSucceed bool) {
ok := acc.Exists(key)
switch {
case !ok && shouldSucceed:
t.Errorf("%s should report key %s as existing", acc.Type(), key)
case ok && !shouldSucceed:
t.Errorf("%s should report key %s as non-existing", acc.Type(), key)
}
}
func testSet(t *testing.T, acc Accessor, key string, shouldSucceed bool, valueToSet interface{}) {
err := acc.Set(key, valueToSet)
switch {
@ -150,7 +160,7 @@ func TestAccessor(t *testing.T) {
testSet(t, acc, "B", true, false)
}
// get again
// get again to check if new values were set
for _, acc := range accs {
testGetString(t, acc, "S", true, "coconut")
testGetInt(t, acc, "I", true, 44)
@ -170,19 +180,69 @@ func TestAccessor(t *testing.T) {
// failures
for _, acc := range accs {
testGetString(t, acc, "S", false, 1)
testGetInt(t, acc, "I", false, 44)
testGetInt(t, acc, "I8", false, 512)
testGetInt(t, acc, "I16", false, 1000000)
testGetInt(t, acc, "I32", false, 44)
testGetInt(t, acc, "I64", false, "44")
testGetInt(t, acc, "UI", false, 44)
testGetInt(t, acc, "UI8", false, 44)
testGetInt(t, acc, "UI16", false, 44)
testGetInt(t, acc, "UI32", false, 44)
testGetInt(t, acc, "UI64", false, 44)
testGetFloat(t, acc, "F32", false, 44.44)
testGetFloat(t, acc, "F64", false, 44.44)
testGetBool(t, acc, "B", false, false)
testSet(t, acc, "S", false, true)
testSet(t, acc, "S", false, false)
testSet(t, acc, "S", false, 1)
testSet(t, acc, "S", false, 1.1)
testSet(t, acc, "I", false, "1")
testSet(t, acc, "I8", false, "1")
testSet(t, acc, "I16", false, "1")
testSet(t, acc, "I32", false, "1")
testSet(t, acc, "I64", false, "1")
testSet(t, acc, "UI", false, "1")
testSet(t, acc, "UI8", false, "1")
testSet(t, acc, "UI16", false, "1")
testSet(t, acc, "UI32", false, "1")
testSet(t, acc, "UI64", false, "1")
testSet(t, acc, "F32", false, "1.1")
testSet(t, acc, "F64", false, "1.1")
testSet(t, acc, "B", false, "false")
testSet(t, acc, "B", false, 1)
testSet(t, acc, "B", false, 1.1)
}
// get again to check if values werent changed when an error occurred
for _, acc := range accs {
testGetString(t, acc, "S", true, "coconut")
testGetInt(t, acc, "I", true, 44)
testGetInt(t, acc, "I8", true, 44)
testGetInt(t, acc, "I16", true, 44)
testGetInt(t, acc, "I32", true, 44)
testGetInt(t, acc, "I64", true, 44)
testGetInt(t, acc, "UI", true, 44)
testGetInt(t, acc, "UI8", true, 44)
testGetInt(t, acc, "UI16", true, 44)
testGetInt(t, acc, "UI32", true, 44)
testGetInt(t, acc, "UI64", true, 44)
testGetFloat(t, acc, "F32", true, 44.44)
testGetFloat(t, acc, "F64", true, 44.44)
testGetBool(t, acc, "B", true, false)
}
// test existence
for _, acc := range accs {
testExists(t, acc, "S", true)
testExists(t, acc, "I", true)
testExists(t, acc, "I8", true)
testExists(t, acc, "I16", true)
testExists(t, acc, "I32", true)
testExists(t, acc, "I64", true)
testExists(t, acc, "UI", true)
testExists(t, acc, "UI8", true)
testExists(t, acc, "UI16", true)
testExists(t, acc, "UI32", true)
testExists(t, acc, "UI64", true)
testExists(t, acc, "F32", true)
testExists(t, acc, "F64", true)
testExists(t, acc, "B", true)
}
// test non-existence
for _, acc := range accs {
testExists(t, acc, "X", false)
}
}

116
database/controller.go
View file

@ -1,7 +1,6 @@
package database
import (
"errors"
"sync"
"time"
@ -29,8 +28,17 @@ func newController(storageInt storage.Interface) (*Controller, error) {
}, nil
}
// ReadOnly returns whether the storage is read only.
func (c *Controller) ReadOnly() bool {
return c.storage.ReadOnly()
}
// Get return the record with the given key.
func (c *Controller) Get(key string) (record.Record, error) {
if shuttingDown.IsSet() {
return nil, ErrShuttingDown
}
r, err := c.storage.Get(key)
if err != nil {
return nil, err
@ -48,101 +56,41 @@ func (c *Controller) Get(key string) (record.Record, error) {
// Put saves a record in the database.
func (c *Controller) Put(r record.Record) error {
if shuttingDown.IsSet() {
return ErrShuttingDown
}
if c.storage.ReadOnly() {
return ErrReadOnly
}
if r.Meta() == nil {
r.SetMeta(&record.Meta{})
}
r.Meta().Update()
return c.storage.Put(r)
}
// Delete a record from the database.
func (c *Controller) Delete(key string) error {
if c.storage.ReadOnly() {
return ErrReadOnly
}
r, err := c.Get(key)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
r.Meta().Deleted = time.Now().Unix()
return c.Put(r)
}
// Partial
// What happens if I mutate a value that does not yet exist? How would I know its type?
func (c *Controller) InsertPartial(key string, partialObject interface{}) error {
if c.storage.ReadOnly() {
return ErrReadOnly
}
return nil
}
func (c *Controller) InsertValue(key string, attribute string, value interface{}) error {
if c.storage.ReadOnly() {
return ErrReadOnly
}
r, err := c.Get(key)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
if r.IsWrapped() {
wrapper, ok := r.(*record.Wrapper)
if !ok {
return errors.New("record is malformed")
}
} else {
}
return nil
}
// Query
// Query executes the given query on the database.
func (c *Controller) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {
return nil, nil
if shuttingDown.IsSet() {
return nil, ErrShuttingDown
}
return c.storage.Query(q, local, internal)
}
// Meta
func (c *Controller) SetAbsoluteExpiry(key string, time int64) error {
if c.storage.ReadOnly() {
return ErrReadOnly
// Maintain runs the Maintain method no the storage.
func (c *Controller) Maintain() error {
return c.storage.Maintain()
}
return nil
// MaintainThorough runs the MaintainThorough method no the storage.
func (c *Controller) MaintainThorough() error {
return c.storage.MaintainThorough()
}
func (c *Controller) SetRelativateExpiry(key string, duration int64) error {
if c.storage.ReadOnly() {
return ErrReadOnly
}
return nil
}
func (c *Controller) MakeCrownJewel(key string) error {
if c.storage.ReadOnly() {
return ErrReadOnly
}
return nil
}
func (c *Controller) MakeSecret(key string) error {
if c.storage.ReadOnly() {
return ErrReadOnly
}
return nil
// Shutdown shuts down the storage.
func (c *Controller) Shutdown() error {
return c.storage.Shutdown()
}

121
database/database.go
View file

@ -1,121 +0,0 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package database
import (
"errors"
)
// Errors
var (
ErrNotFound = errors.New("database entry could not be found")
ErrPermissionDenied = errors.New("access to database record denied")
ErrReadOnly = errors.New("database is read only")
)
func init() {
// if strings.HasSuffix(os.Args[0], ".test") {
// // testing setup
// log.Warning("===== DATABASE RUNNING IN TEST MODE =====")
// db = channelshim.NewChanneledDatastore(ds.NewMapDatastore())
// return
// }
// sfsDB, err := simplefs.NewDatastore(meta.DatabaseDir())
// if err != nil {
// fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err)
// os.Exit(1)
// }
// ldb, err := leveldb.NewDatastore(path.Join(meta.DatabaseDir(), "leveldb"), &leveldb.Options{})
// if err != nil {
// fmt.Fprintf(os.Stderr, "FATAL ERROR: could not init simplefs database: %s\n", err)
// os.Exit(1)
// }
//
// mapDB := ds.NewMapDatastore()
//
// db = channelshim.NewChanneledDatastore(mount.New([]mount.Mount{
// mount.Mount{
// Prefix: ds.NewKey("/Run"),
// Datastore: mapDB,
// },
// mount.Mount{
// Prefix: ds.NewKey("/"),
// Datastore: ldb,
// },
// }))
}
// func Batch() (ds.Batch, error) {
// return db.Batch()
// }
// func Close() error {
// return db.Close()
// }
// func Get(key *ds.Key) (Model, error) {
// data, err := db.Get(*key)
// if err != nil {
// switch err {
// case ds.ErrNotFound:
// return nil, ErrNotFound
// default:
// return nil, err
// }
// }
// model, ok := data.(Model)
// if !ok {
// return nil, errors.New("database did not return model")
// }
// return model, nil
// }
// func Has(key ds.Key) (exists bool, err error) {
// return db.Has(key)
// }
//
// func Create(key ds.Key, model Model) (err error) {
// handleCreateSubscriptions(model)
// err = db.Put(key, model)
// if err != nil {
// log.Tracef("database: failed to create entry %s: %s", key, err)
// }
// return err
// }
//
// func Update(key ds.Key, model Model) (err error) {
// handleUpdateSubscriptions(model)
// err = db.Put(key, model)
// if err != nil {
// log.Tracef("database: failed to update entry %s: %s", key, err)
// }
// return err
// }
//
// func Delete(key ds.Key) (err error) {
// handleDeleteSubscriptions(&key)
// return db.Delete(key)
// }
//
// func Query(q dsq.Query) (dsq.Results, error) {
// return db.Query(q)
// }
//
// func RawGet(key ds.Key) (*dbutils.Wrapper, error) {
// data, err := db.Get(key)
// if err != nil {
// return nil, err
// }
// wrapped, ok := data.(*dbutils.Wrapper)
// if !ok {
// return nil, errors.New("returned data is not a wrapper")
// }
// return wrapped, nil
// }
//
// func RawPut(key ds.Key, value interface{}) error {
// return db.Put(key, value)
// }

69
database/database_test.go Normal file
View file

@ -0,0 +1,69 @@
package database
import (
"io/ioutil"
"os"
"sync"
"testing"
"github.com/Safing/portbase/database/record"
)
type TestRecord struct {
record.Base
lock sync.Mutex
S string
I int
I8 int8
I16 int16
I32 int32
I64 int64
UI uint
UI8 uint8
UI16 uint16
UI32 uint32
UI64 uint64
F32 float32
F64 float64
B bool
}
func (tr *TestRecord) Lock() {
}
func (tr *TestRecord) Unlock() {
}
func TestDatabase(t *testing.T) {
testDir, err := ioutil.TempDir("", "testing-")
if err != nil {
t.Fatal(err)
}
err = Initialize(testDir)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(testDir) // clean up
err = RegisterDatabase(&RegisteredDatabase{
Name: "testing",
Description: "Unit Test Database",
StorageType: "badger",
PrimaryAPI: "",
})
if err != nil {
t.Fatal(err)
}
db := NewInterface(nil)
new := &TestRecord{}
new.SetKey("testing:A")
err = db.Put(new)
if err != nil {
t.Fatal(err)
}
}

7
database/databases.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"path"
"github.com/tevino/abool"
"github.com/Safing/portbase/database/storage"
"github.com/Safing/portbase/database/record"
)
@ -13,14 +14,16 @@ import (
var (
databases = make(map[string]*Controller)
databasesLock sync.Mutex
shuttingDown = abool.NewBool(false)
)
func splitKeyAndGetDatabase(key string) (dbKey string, db *Controller, err error) {
func splitKeyAndGetDatabase(key string) (db *Controller, dbKey string, err error) {
var dbName string
dbName, dbKey = record.ParseKey(key)
db, err = getDatabase(dbName)
if err != nil {
return "", nil, err
return nil, "", err
}
return
}

18
database/dbmodule/db.go
View file

@ -1,11 +1,18 @@
package dbmodule
import (
"errors"
"flag"
"sync"
"github.com/Safing/portbase/database"
"github.com/Safing/portbase/modules"
)
var (
databaseDir string
shutdownSignal = make(chan struct{})
maintenanceWg sync.WaitGroup
)
func init() {
@ -18,12 +25,19 @@ func prep() error {
if databaseDir == "" {
return errors.New("no database location specified, set with `-db=/path/to/db`")
}
return nil
}
func start() error {
return database.Initialize(databaseDir)
err := database.Initialize(databaseDir)
if err == nil {
go maintainer()
}
return err
}
func stop() {
func stop() error {
close(shutdownSignal)
maintenanceWg.Wait()
return database.Shutdown()
}

32
database/dbmodule/maintenance.go Normal file
View file

@ -0,0 +1,32 @@
package dbmodule
import (
"time"
"github.com/Safing/portbase/database"
"github.com/Safing/portbase/log"
)
func maintainer() {
ticker := time.NewTicker(1 * time.Hour)
tickerThorough := time.NewTicker(10 * time.Minute)
maintenanceWg.Add(1)
for {
select {
case <- ticker.C:
err := database.Maintain()
if err != nil {
log.Errorf("database: maintenance error: %s", err)
}
case <- ticker.C:
err := database.MaintainThorough()
if err != nil {
log.Errorf("database: maintenance (thorough) error: %s", err)
}
case <-shutdownSignal:
maintenanceWg.Done()
return
}
}
}

15
database/errors.go Normal file
View file

@ -0,0 +1,15 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package database
import (
"errors"
)
// Errors
var (
ErrNotFound = errors.New("database entry could not be found")
ErrPermissionDenied = errors.New("access to database record denied")
ErrReadOnly = errors.New("database is read only")
ErrShuttingDown = errors.New("database system is shutting down")
)

185
database/interface.go
View file

@ -1,9 +1,19 @@
package database
import (
"errors"
"fmt"
"github.com/Safing/portbase/database/accessor"
"github.com/Safing/portbase/database/iterator"
"github.com/Safing/portbase/database/query"
"github.com/Safing/portbase/database/record"
)
const (
getDBFromKey = ""
)
// Interface provides a method to access the database with attached options.
type Interface struct {
options *Options
@ -30,7 +40,7 @@ func NewInterface(opts *Options) *Interface {
// Exists return whether a record with the given key exists.
func (i *Interface) Exists(key string) (bool, error) {
_, err := i.getRecord(key)
_, _, err := i.getRecord(getDBFromKey, key, false, false)
if err != nil {
if err == ErrNotFound {
return false, nil
@ -42,28 +52,161 @@ func (i *Interface) Exists(key string) (bool, error) {
// Get return the record with the given key.
func (i *Interface) Get(key string) (record.Record, error) {
r, err := i.getRecord(key)
r, _, err := i.getRecord(getDBFromKey, key, true, false)
return r, err
}
func (i *Interface) getRecord(dbName string, dbKey string, check bool, mustBeWriteable bool) (r record.Record, db *Controller, err error) {
if dbName == "" {
dbName, dbKey = record.ParseKey(dbKey)
}
db, err = getDatabase(dbName)
if err != nil {
return nil, nil, err
}
if mustBeWriteable && db.ReadOnly() {
return nil, nil, ErrReadOnly
}
r, err = db.Get(dbKey)
if err != nil {
return nil, nil, err
}
if check && !r.Meta().CheckPermission(i.options.Local, i.options.Internal) {
return nil, nil, ErrPermissionDenied
}
return r, db, nil
}
// InsertValue inserts a value into a record.
func (i *Interface) InsertValue(key string, attribute string, value interface{}) error {
r, db, err := i.getRecord(getDBFromKey, key, true, true)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
var acc accessor.Accessor
if r.IsWrapped() {
wrapper, ok := r.(*record.Wrapper)
if !ok {
return errors.New("record is malformed (reports to be wrapped but is not of type *record.Wrapper)")
}
acc = accessor.NewJSONBytesAccessor(&wrapper.Data)
} else {
acc = accessor.NewStructAccessor(r)
}
err = acc.Set(attribute, value)
if err != nil {
return fmt.Errorf("failed to set value with %s: %s", acc.Type(), err)
}
return db.Put(r)
}
// Put saves a record to the database.
func (i *Interface) Put(r record.Record) error {
_, db, err := i.getRecord(r.DatabaseName(), r.DatabaseKey(), true, true)
if err != nil {
return err
}
return db.Put(r)
}
// PutNew saves a record to the database as a new record (ie. with a new creation timestamp).
func (i *Interface) PutNew(r record.Record) error {
_, db, err := i.getRecord(r.DatabaseName(), r.DatabaseKey(), true, true)
if err != nil && err != ErrNotFound {
return err
}
r.SetMeta(&record.Meta{})
return db.Put(r)
}
// SetAbsoluteExpiry sets an absolute record expiry.
func (i *Interface) SetAbsoluteExpiry(key string, time int64) error {
r, db, err := i.getRecord(getDBFromKey, key, true, true)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
r.Meta().SetAbsoluteExpiry(time)
return db.Put(r)
}
// SetRelativateExpiry sets a relative (self-updating) record expiry.
func (i *Interface) SetRelativateExpiry(key string, duration int64) error {
r, db, err := i.getRecord(getDBFromKey, key, true, true)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
r.Meta().SetRelativateExpiry(duration)
return db.Put(r)
}
// MakeSecret marks the record as a secret, meaning interfacing processes, such as an UI, are denied access to the record.
func (i *Interface) MakeSecret(key string) error {
r, db, err := i.getRecord(getDBFromKey, key, true, true)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
r.Meta().MakeSecret()
return db.Put(r)
}
// MakeCrownJewel marks a record as a crown jewel, meaning it will only be accessible locally.
func (i *Interface) MakeCrownJewel(key string) error {
r, db, err := i.getRecord(getDBFromKey, key, true, true)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
r.Meta().MakeCrownJewel()
return db.Put(r)
}
// Delete deletes a record from the database.
func (i *Interface) Delete(key string) error {
r, db, err := i.getRecord(getDBFromKey, key, true, true)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
r.Meta().Delete()
return db.Put(r)
}
// Query executes the given query on the database.
func (i *Interface) Query(q *query.Query) (*iterator.Iterator, error) {
db, err := getDatabase(q.DatabaseName())
if err != nil {
return nil, err
}
if !r.Meta().CheckPermission(i.options.Local, i.options.Internal) {
return nil, ErrPermissionDenied
}
return r, nil
}
func (i *Interface) getRecord(key string) (record.Record, error) {
dbKey, db, err := splitKeyAndGetDatabase(key)
if err != nil {
return nil, err
}
r, err := db.Get(dbKey)
if err != nil {
return nil, err
}
return r, nil
return db.Query(q, i.options.Local, i.options.Internal)
}

45
database/location.go
View file

@ -2,12 +2,57 @@ package database
import (
"path"
"os"
"fmt"
"errors"
)
var (
rootDir string
)
// Initialize initialized the database
func Initialize(location string) error {
if initialized.SetToIf(false, true) {
rootDir = location
err := checkRootDir()
if err != nil {
return fmt.Errorf("could not create/open database directory (%s): %s", rootDir, err)
}
err = loadRegistry()
if err != nil {
return fmt.Errorf("could not load database registry (%s): %s", path.Join(rootDir, registryFileName), err)
}
return nil
}
return errors.New("database already initialized")
}
func checkRootDir() error {
// open dir
dir, err := os.Open(rootDir)
if err != nil {
if err == os.ErrNotExist {
return os.MkdirAll(rootDir, 0700)
}
return err
}
defer dir.Close()
fileInfo, err := dir.Stat()
if err != nil {
return err
}
if fileInfo.Mode().Perm() != 0700 {
return dir.Chmod(0700)
}
return nil
}
// getLocation returns the storage location for the given name and type.
func getLocation(name, storageType string) (location string, err error) {
return path.Join(rootDir, name, storageType), nil

50
database/maintainence.go Normal file
View file

@ -0,0 +1,50 @@
package database
// Maintain runs the Maintain method on all storages.
func Maintain() (err error) {
controllers := duplicateControllers()
for _, c := range controllers {
err = c.Maintain()
if err != nil {
return
}
}
return
}
// MaintainThorough runs the MaintainThorough method on all storages.
func MaintainThorough() (err error) {
controllers := duplicateControllers()
for _, c := range controllers {
err = c.MaintainThorough()
if err != nil {
return
}
}
return
}
// Shutdown shuts down the whole database system.
func Shutdown() (err error) {
shuttingDown.Set()
controllers := duplicateControllers()
for _, c := range controllers {
err = c.Shutdown()
if err != nil {
return
}
}
return
}
func duplicateControllers() (controllers []*Controller) {
databasesLock.Lock()
defer databasesLock.Unlock()
for _, c := range databases {
controllers = append(controllers, c)
}
return
}

6
database/query/condition-and.go
View file

@ -3,6 +3,8 @@ package query
import (
"fmt"
"strings"
"github.com/Safing/portbase/database/accessor"
)
// And combines multiple conditions with a logical _AND_ operator.
@ -16,9 +18,9 @@ type andCond struct {
conditions []Condition
}
func (c *andCond) complies(f Fetcher) bool {
func (c *andCond) complies(acc accessor.Accessor) bool {
for _, cond := range c.conditions {
if !cond.complies(f) {
if !cond.complies(acc) {
return false
}
}

6
database/query/condition-bool.go
View file

@ -4,6 +4,8 @@ import (
"errors"
"fmt"
"strconv"
"github.com/Safing/portbase/database/accessor"
)
type boolCondition struct {
@ -42,8 +44,8 @@ func newBoolCondition(key string, operator uint8, value interface{}) *boolCondit
}
}
func (c *boolCondition) complies(f Fetcher) bool {
comp, ok := f.GetBool(c.key)
func (c *boolCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetBool(c.key)
if !ok {
return false
}

6
database/query/condition-error.go
View file

@ -1,5 +1,9 @@
package query
import (
"github.com/Safing/portbase/database/accessor"
)
type errorCondition struct {
err error
}
@ -10,7 +14,7 @@ func newErrorCondition(err error) *errorCondition {
}
}
func (c *errorCondition) complies(f Fetcher) bool {
func (c *errorCondition) complies(acc accessor.Accessor) bool {
return false
}

6
database/query/condition-exists.go
View file

@ -3,6 +3,8 @@ package query
import (
"errors"
"fmt"
"github.com/Safing/portbase/database/accessor"
)
type existsCondition struct {
@ -17,8 +19,8 @@ func newExistsCondition(key string, operator uint8) *existsCondition {
}
}
func (c *existsCondition) complies(f Fetcher) bool {
return f.Exists(c.key)
func (c *existsCondition) complies(acc accessor.Accessor) bool {
return acc.Exists(c.key)
}
func (c *existsCondition) check() error {

6
database/query/condition-float.go
View file

@ -4,6 +4,8 @@ import (
"errors"
"fmt"
"strconv"
"github.com/Safing/portbase/database/accessor"
)
type floatCondition struct {
@ -62,8 +64,8 @@ func newFloatCondition(key string, operator uint8, value interface{}) *floatCond
}
}
func (c *floatCondition) complies(f Fetcher) bool {
comp, ok := f.GetFloat(c.key)
func (c *floatCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetFloat(c.key)
if !ok {
return false
}

6
database/query/condition-int.go
View file

@ -4,6 +4,8 @@ import (
"errors"
"fmt"
"strconv"
"github.com/Safing/portbase/database/accessor"
)
type intCondition struct {
@ -58,8 +60,8 @@ func newIntCondition(key string, operator uint8, value interface{}) *intConditio
}
}
func (c *intCondition) complies(f Fetcher) bool {
comp, ok := f.GetInt(c.key)
func (c *intCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetInt(c.key)
if !ok {
return false
}

6
database/query/condition-no.go
View file

@ -1,9 +1,13 @@
package query
import (
"github.com/Safing/portbase/database/accessor"
)
type noCond struct {
}
func (c *noCond) complies(f Fetcher) bool {
func (c *noCond) complies(acc accessor.Accessor) bool {
return true
}

6
database/query/condition-not.go
View file

@ -3,6 +3,8 @@ package query
import (
"fmt"
"strings"
"github.com/Safing/portbase/database/accessor"
)
// Not negates the supplied condition.
@ -16,8 +18,8 @@ type notCond struct {
notC Condition
}
func (c *notCond) complies(f Fetcher) bool {
return !c.notC.complies(f)
func (c *notCond) complies(acc accessor.Accessor) bool {
return !c.notC.complies(acc)
}
func (c *notCond) check() error {

6
database/query/condition-or.go
View file

@ -3,6 +3,8 @@ package query
import (
"fmt"
"strings"
"github.com/Safing/portbase/database/accessor"
)
// Or combines multiple conditions with a logical _OR_ operator.
@ -16,9 +18,9 @@ type orCond struct {
conditions []Condition
}
func (c *orCond) complies(f Fetcher) bool {
func (c *orCond) complies(acc accessor.Accessor) bool {
for _, cond := range c.conditions {
if cond.complies(f) {
if cond.complies(acc) {
return true
}
}

6
database/query/condition-regex.go
View file

@ -4,6 +4,8 @@ import (
"errors"
"fmt"
"regexp"
"github.com/Safing/portbase/database/accessor"
)
type regexCondition struct {
@ -35,8 +37,8 @@ func newRegexCondition(key string, operator uint8, value interface{}) *regexCond
}
}
func (c *regexCondition) complies(f Fetcher) bool {
comp, ok := f.GetString(c.key)
func (c *regexCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetString(c.key)
if !ok {
return false
}

6
database/query/condition-string.go
View file

@ -4,6 +4,8 @@ import (
"errors"
"fmt"
"strings"
"github.com/Safing/portbase/database/accessor"
)
type stringCondition struct {
@ -28,8 +30,8 @@ func newStringCondition(key string, operator uint8, value interface{}) *stringCo
}
}
func (c *stringCondition) complies(f Fetcher) bool {
comp, ok := f.GetString(c.key)
func (c *stringCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetString(c.key)
if !ok {
return false
}

5
database/query/condition-stringslice.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"strings"
"github.com/Safing/portbase/database/accessor"
"github.com/Safing/portbase/utils"
)
@ -44,8 +45,8 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
}
func (c *stringSliceCondition) complies(f Fetcher) bool {
comp, ok := f.GetString(c.key)
func (c *stringSliceCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetString(c.key)
if !ok {
return false
}

8
database/query/condition.go
View file

@ -1,10 +1,14 @@
package query
import "fmt"
import (
"fmt"
"github.com/Safing/portbase/database/accessor"
)
// Condition is an interface to provide a common api to all condition types.
type Condition interface {
complies(f Fetcher) bool
complies(acc accessor.Accessor) bool
check() error
string() string
}

1
database/query/parser_test.go
View file

@ -144,7 +144,6 @@ func testParseError(t *testing.T, queryText string, expectedErrorString string)
func TestParseErrors(t *testing.T) {
// syntax
testParseError(t, `query`, `unexpected end at position 5`)
testParseError(t, `query test`, `invalid prefix: test`)
testParseError(t, `query test: where`, `unexpected end at position 17`)
testParseError(t, `query test: where (`, `unexpected end at position 19`)
testParseError(t, `query test: where )`, `unknown clause ")" at position 19`)

36
database/query/query.go
View file

@ -2,8 +2,10 @@ package query
import (
"fmt"
"regexp"
"strings"
"github.com/Safing/portbase/database/accessor"
"github.com/Safing/portbase/database/record"
)
// Example:
@ -16,14 +18,11 @@ import (
// )
// )
var (
prefixExpr = regexp.MustCompile("^[a-z-]+:")
)
// Query contains a compiled query.
type Query struct {
checked bool
prefix string
dbName string
dbKeyPrefix string
where Condition
orderBy string
limit int
@ -32,8 +31,10 @@ type Query struct {
// New creates a new query with the supplied prefix.
func New(prefix string) *Query {
dbName, dbKeyPrefix := record.ParseKey(prefix)
return &Query{
prefix: prefix,
dbName: dbName,
dbKeyPrefix: dbKeyPrefix,
}
}
@ -67,11 +68,6 @@ func (q *Query) Check() (*Query, error) {
return q, nil
}
// check prefix
if !prefixExpr.MatchString(q.prefix) {
return nil, fmt.Errorf("invalid prefix: %s", q.prefix)
}
// check condition
if q.where != nil {
err := q.where.check()
@ -101,8 +97,8 @@ func (q *Query) IsChecked() bool {
}
// Matches checks whether the query matches the supplied data object.
func (q *Query) Matches(f Fetcher) bool {
return q.where.complies(f)
func (q *Query) Matches(acc accessor.Accessor) bool {
return q.where.complies(acc)
}
// Print returns the string representation of the query.
@ -130,5 +126,15 @@ func (q *Query) Print() string {
offset = fmt.Sprintf(" offset %d", q.offset)
}
return fmt.Sprintf("query %s%s%s%s%s", q.prefix, where, orderBy, limit, offset)
return fmt.Sprintf("query %s:%s%s%s%s%s", q.dbName, q.dbKeyPrefix, where, orderBy, limit, offset)
}
// DatabaseName returns the name of the database.
func (q *Query) DatabaseName() string {
return q.dbName
}
// DatabaseKeyPrefix returns the key prefix for the database.
func (q *Query) DatabaseKeyPrefix() string {
return q.dbKeyPrefix
}

8
database/query/query_test.go
View file

@ -2,6 +2,8 @@ package query
import (
"testing"
"github.com/Safing/portbase/database/accessor"
)
var (
@ -44,12 +46,12 @@ var (
}`
)
func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) {
func testQuery(t *testing.T, acc accessor.Accessor, shouldMatch bool, condition Condition) {
q := New("test:").Where(condition).MustBeValid()
// fmt.Printf("%s\n", q.String())
matched := q.Matches(f)
matched := q.Matches(acc)
switch {
case !matched && shouldMatch:
t.Errorf("should match: %s", q.Print())
@ -63,7 +65,7 @@ func TestQuery(t *testing.T) {
// if !gjson.Valid(testJSON) {
// t.Fatal("test json is invalid")
// }
f := NewJSONFetcher(testJSON)
f := accessor.NewJSONAccessor(&testJSON)
testQuery(t, f, true, Where("age", Equals, 100))
testQuery(t, f, true, Where("age", GreaterThan, uint8(99)))

5
database/record/meta.go
View file

@ -73,6 +73,11 @@ func (m *Meta) Reset() {
m.Deleted = 0
}
// Delete marks the record as deleted.
func (m *Meta) Delete() {
m.Deleted = time.Now().Unix()
}
// CheckValidity checks whether the database record is valid.
func (m *Meta) CheckValidity(now int64) (valid bool) {
switch {

50
database/registry.go
View file

@ -3,10 +3,10 @@ package database
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"regexp"
"sync"
"github.com/tevino/abool"
@ -40,6 +40,8 @@ var (
registry map[string]*RegisteredDatabase
registryLock sync.Mutex
nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{5,}$")
)
// RegisterDatabase registers a new database.
@ -48,6 +50,10 @@ func RegisterDatabase(new *RegisteredDatabase) error {
return errors.New("database not initialized")
}
if !nameConstraint.MatchString(new.Name) {
return errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 5 characters long")
}
registryLock.Lock()
defer registryLock.Unlock()
@ -60,48 +66,6 @@ func RegisterDatabase(new *RegisteredDatabase) error {
return nil
}
// Initialize initialized the database
func Initialize(location string) error {
if initialized.SetToIf(false, true) {
rootDir = location
err := checkRootDir()
if err != nil {
return fmt.Errorf("could not create/open database directory (%s): %s", rootDir, err)
}
err = loadRegistry()
if err != nil {
return fmt.Errorf("could not load database registry (%s): %s", path.Join(rootDir, registryFileName), err)
}
return nil
}
return errors.New("database already initialized")
}
func checkRootDir() error {
// open dir
dir, err := os.Open(rootDir)
if err != nil {
if err == os.ErrNotExist {
return os.MkdirAll(rootDir, 0700)
}
return err
}
defer dir.Close()
fileInfo, err := dir.Stat()
if err != nil {
return err
}
if fileInfo.Mode().Perm() != 0700 {
return dir.Chmod(0700)
}
return nil
}
func loadRegistry() error {
registryLock.Lock()
defer registryLock.Unlock()

2
database/storage/interface.go
View file

@ -11,7 +11,7 @@ type Interface interface {
Get(key string) (record.Record, error)
Put(m record.Record) error
Delete(key string) error
Query(q *query.Query) (*iterator.Iterator, error)
Query(q *query.Query, local, internal bool) (*iterator.Iterator, error)
ReadOnly() bool
Maintain() error