Revamp feeder to accessor

database/record/accessor-json-bytes.go

@@ -0,0 +1,81 @@
+package record
+
+import (
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+// JSONBytesAccessor is a json string with get functions.
+type JSONBytesAccessor struct {
+	json *[]byte
+}
+
+// NewJSONBytesAccessor adds the Accessor interface to a JSON bytes string.
+func NewJSONBytesAccessor(json *[]byte) *JSONBytesAccessor {
+	return &JSONBytesAccessor{
+		json: json,
+	}
+}
+
+// Set sets the value identified by key.
+func (ja *JSONBytesAccessor) Set(key string, value interface{}) error {
+	new, err := sjson.SetBytes(*ja.json, key, value)
+	if err != nil {
+		return err
+	}
+	*ja.json = new
+	return nil
+}
+
+// GetString returns the string found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetString(key string) (value string, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.String {
+		return emptyString, false
+	}
+	return result.String(), true
+}
+
+// GetInt returns the int found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetInt(key string) (value int64, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Int(), true
+}
+
+// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetFloat(key string) (value float64, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Float(), true
+}
+
+// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
+func (ja *JSONBytesAccessor) GetBool(key string) (value bool, ok bool) {
+	result := gjson.GetBytes(*ja.json, key)
+	switch {
+	case !result.Exists():
+		return false, false
+	case result.Type == gjson.True:
+		return true, true
+	case result.Type == gjson.False:
+		return false, true
+	default:
+		return false, false
+	}
+}
+
+// Exists returns the whether the given key exists.
+func (ja *JSONBytesAccessor) Exists(key string) bool {
+	result := gjson.GetBytes(*ja.json, key)
+	return result.Exists()
+}
+
+// Type returns the accessor type as a string.
+func (ja *JSONBytesAccessor) Type() string {
+	return "JSONBytesAccessor"
+}

database/record/accessor-json-string.go

@@ -0,0 +1,81 @@
+package record
+
+import (
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+// JSONAccessor is a json string with get functions.
+type JSONAccessor struct {
+	json *string
+}
+
+// NewJSONAccessor adds the Accessor interface to a JSON string.
+func NewJSONAccessor(json *string) *JSONAccessor {
+	return &JSONAccessor{
+		json: json,
+	}
+}
+
+// Set sets the value identified by key.
+func (ja *JSONAccessor) Set(key string, value interface{}) error {
+	new, err := sjson.Set(*ja.json, key, value)
+	if err != nil {
+		return err
+	}
+	*ja.json = new
+	return nil
+}
+
+// GetString returns the string found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetString(key string) (value string, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.String {
+		return emptyString, false
+	}
+	return result.String(), true
+}
+
+// GetInt returns the int found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetInt(key string) (value int64, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Int(), true
+}
+
+// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetFloat(key string) (value float64, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	if !result.Exists() || result.Type != gjson.Number {
+		return 0, false
+	}
+	return result.Float(), true
+}
+
+// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
+func (ja *JSONAccessor) GetBool(key string) (value bool, ok bool) {
+	result := gjson.Get(*ja.json, key)
+	switch {
+	case !result.Exists():
+		return false, false
+	case result.Type == gjson.True:
+		return true, true
+	case result.Type == gjson.False:
+		return false, true
+	default:
+		return false, false
+	}
+}
+
+// Exists returns the whether the given key exists.
+func (ja *JSONAccessor) Exists(key string) bool {
+	result := gjson.Get(*ja.json, key)
+	return result.Exists()
+}
+
+// Type returns the accessor type as a string.
+func (ja *JSONAccessor) Type() string {
+	return "JSONAccessor"
+}

database/record/accessor-struct.go

@@ -0,0 +1,149 @@
+package record
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+)
+
+// StructAccessor is a json string with get functions.
+type StructAccessor struct {
+	object reflect.Value
+}
+
+// NewStructAccessor adds the Accessor interface to a JSON string.
+func NewStructAccessor(object interface{}) *StructAccessor {
+	return &StructAccessor{
+		object: reflect.ValueOf(object).Elem(),
+	}
+}
+
+// Set sets the value identified by key.
+func (sa *StructAccessor) Set(key string, value interface{}) error {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() {
+		return errors.New("struct field does not exist")
+	}
+	if !field.CanSet() {
+		return fmt.Errorf("field %s or struct is immutable", field.String())
+	}
+
+	newVal := reflect.ValueOf(value)
+
+	// set directly if type matches
+	if newVal.Kind() == field.Kind() {
+		field.Set(newVal)
+		return nil
+	}
+
+	// handle special cases
+	switch field.Kind() {
+
+	// ints
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		var newInt int64
+		switch newVal.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			newInt = newVal.Int()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			newInt = int64(newVal.Uint())
+		default:
+			return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+		}
+		if field.OverflowInt(newInt) {
+			return fmt.Errorf("setting field %s (%s) to %d would overflow", key, field.Kind().String(), newInt)
+		}
+		field.SetInt(newInt)
+
+		// uints
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		var newUint uint64
+		switch newVal.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			newUint = uint64(newVal.Int())
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			newUint = newVal.Uint()
+		default:
+			return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+		}
+		if field.OverflowUint(newUint) {
+			return fmt.Errorf("setting field %s (%s) to %d would overflow", key, field.Kind().String(), newUint)
+		}
+		field.SetUint(newUint)
+
+		// floats
+	case reflect.Float32, reflect.Float64:
+		switch newVal.Kind() {
+		case reflect.Float32, reflect.Float64:
+			field.SetFloat(newVal.Float())
+		default:
+			return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+		}
+	default:
+		return fmt.Errorf("tried to set field %s (%s) to a %s value", key, field.Kind().String(), newVal.Kind().String())
+	}
+
+	return nil
+}
+
+// GetString returns the string found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetString(key string) (value string, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() || field.Kind() != reflect.String {
+		return "", false
+	}
+	return field.String(), true
+}
+
+// GetInt returns the int found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetInt(key string) (value int64, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() {
+		return 0, false
+	}
+	switch field.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return field.Int(), true
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return int64(field.Uint()), true
+	default:
+		return 0, false
+	}
+}
+
+// GetFloat returns the float found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetFloat(key string) (value float64, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() {
+		return 0, false
+	}
+	switch field.Kind() {
+	case reflect.Float32, reflect.Float64:
+		return field.Float(), true
+	default:
+		return 0, false
+	}
+}
+
+// GetBool returns the bool found by the given json key and whether it could be successfully extracted.
+func (sa *StructAccessor) GetBool(key string) (value bool, ok bool) {
+	field := sa.object.FieldByName(key)
+	if !field.IsValid() || field.Kind() != reflect.Bool {
+		return false, false
+	}
+	return field.Bool(), true
+}
+
+// Exists returns the whether the given key exists.
+func (sa *StructAccessor) Exists(key string) bool {
+	field := sa.object.FieldByName(key)
+	if field.IsValid() {
+		return true
+	}
+	return false
+}
+
+// Type returns the accessor type as a string.
+func (sa *StructAccessor) Type() string {
+	return "StructAccessor"
+}

database/record/accessor.go

@@ -0,0 +1,18 @@
+package record
+
+const (
+	emptyString = ""
+)
+
+// Accessor provides an interface to supply the query matcher a method to retrieve values from an object.
+type Accessor interface {
+	GetString(key string) (value string, ok bool)
+	GetInt(key string) (value int64, ok bool)
+	GetFloat(key string) (value float64, ok bool)
+	GetBool(key string) (value bool, ok bool)
+	Exists(key string) bool
+
+	Set(key string, value interface{}) error
+
+	Type() string
+}

database/record/accessor_test.go

@@ -0,0 +1,188 @@
+package record
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+type TestStruct struct {
+	S    string
+	I    int
+	I8   int8
+	I16  int16
+	I32  int32
+	I64  int64
+	UI   uint
+	UI8  uint8
+	UI16 uint16
+	UI32 uint32
+	UI64 uint64
+	F32  float32
+	F64  float64
+	B    bool
+}
+
+var (
+	testStruct = &TestStruct{
+		S:    "banana",
+		I:    42,
+		I8:   42,
+		I16:  42,
+		I32:  42,
+		I64:  42,
+		UI:   42,
+		UI8:  42,
+		UI16: 42,
+		UI32: 42,
+		UI64: 42,
+		F32:  42.42,
+		F64:  42.42,
+		B:    true,
+	}
+	testJSONBytes, _ = json.Marshal(testStruct)
+	testJSON         = string(testJSONBytes)
+)
+
+func testGetString(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue string) {
+	v, ok := acc.GetString(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get string with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get string with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if v != expectedValue {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testGetInt(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue int64) {
+	v, ok := acc.GetInt(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get int with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get int with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if v != expectedValue {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testGetFloat(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue float64) {
+	v, ok := acc.GetFloat(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get float with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get float with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if int64(v) != int64(expectedValue) {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testGetBool(t *testing.T, acc Accessor, key string, shouldSucceed bool, expectedValue bool) {
+	v, ok := acc.GetBool(key)
+	switch {
+	case !ok && shouldSucceed:
+		t.Errorf("%s failed to get bool with key %s", acc.Type(), key)
+	case ok && !shouldSucceed:
+		t.Errorf("%s should have failed to get bool with key %s, it returned %v", acc.Type(), key, v)
+	}
+	if v != expectedValue {
+		t.Errorf("%s returned an unexpected value: wanted %v, got %v", acc.Type(), expectedValue, v)
+	}
+}
+
+func testSet(t *testing.T, acc Accessor, key string, shouldSucceed bool, valueToSet interface{}) {
+	err := acc.Set(key, valueToSet)
+	switch {
+	case err != nil && shouldSucceed:
+		t.Errorf("%s failed to set %s to %+v: %s", acc.Type(), key, valueToSet, err)
+	case err == nil && !shouldSucceed:
+		t.Errorf("%s should have failed to set %s to %+v", acc.Type(), key, valueToSet)
+	}
+}
+
+func TestAccessor(t *testing.T) {
+
+	// Test interface compliance
+	accs := []Accessor{
+		NewJSONAccessor(&testJSON),
+		NewJSONBytesAccessor(&testJSONBytes),
+		NewStructAccessor(testStruct),
+	}
+
+	// get
+	for _, acc := range accs {
+		testGetString(t, acc, "S", true, "banana")
+		testGetInt(t, acc, "I", true, 42)
+		testGetInt(t, acc, "I8", true, 42)
+		testGetInt(t, acc, "I16", true, 42)
+		testGetInt(t, acc, "I32", true, 42)
+		testGetInt(t, acc, "I64", true, 42)
+		testGetInt(t, acc, "UI", true, 42)
+		testGetInt(t, acc, "UI8", true, 42)
+		testGetInt(t, acc, "UI16", true, 42)
+		testGetInt(t, acc, "UI32", true, 42)
+		testGetInt(t, acc, "UI64", true, 42)
+		testGetFloat(t, acc, "F32", true, 42.42)
+		testGetFloat(t, acc, "F64", true, 42.42)
+		testGetBool(t, acc, "B", true, true)
+	}
+
+	// set
+	for _, acc := range accs {
+		testSet(t, acc, "S", true, "coconut")
+		testSet(t, acc, "I", true, uint32(44))
+		testSet(t, acc, "I8", true, uint64(44))
+		testSet(t, acc, "I16", true, uint8(44))
+		testSet(t, acc, "I32", true, uint16(44))
+		testSet(t, acc, "I64", true, 44)
+		testSet(t, acc, "UI", true, 44)
+		testSet(t, acc, "UI8", true, int64(44))
+		testSet(t, acc, "UI16", true, int32(44))
+		testSet(t, acc, "UI32", true, int8(44))
+		testSet(t, acc, "UI64", true, int16(44))
+		testSet(t, acc, "F32", true, 44.44)
+		testSet(t, acc, "F64", true, 44.44)
+		testSet(t, acc, "B", true, false)
+	}
+
+	// get again
+	for _, acc := range accs {
+		testGetString(t, acc, "S", true, "coconut")
+		testGetInt(t, acc, "I", true, 44)
+		testGetInt(t, acc, "I8", true, 44)
+		testGetInt(t, acc, "I16", true, 44)
+		testGetInt(t, acc, "I32", true, 44)
+		testGetInt(t, acc, "I64", true, 44)
+		testGetInt(t, acc, "UI", true, 44)
+		testGetInt(t, acc, "UI8", true, 44)
+		testGetInt(t, acc, "UI16", true, 44)
+		testGetInt(t, acc, "UI32", true, 44)
+		testGetInt(t, acc, "UI64", true, 44)
+		testGetFloat(t, acc, "F32", true, 44.44)
+		testGetFloat(t, acc, "F64", true, 44.44)
+		testGetBool(t, acc, "B", true, false)
+	}
+
+	// failures
+	for _, acc := range accs {
+		testGetString(t, acc, "S", false, 1)
+		testGetInt(t, acc, "I", false, 44)
+		testGetInt(t, acc, "I8", false, 512)
+		testGetInt(t, acc, "I16", false, 1000000)
+		testGetInt(t, acc, "I32", false, 44)
+		testGetInt(t, acc, "I64", false, "44")
+		testGetInt(t, acc, "UI", false, 44)
+		testGetInt(t, acc, "UI8", false, 44)
+		testGetInt(t, acc, "UI16", false, 44)
+		testGetInt(t, acc, "UI32", false, 44)
+		testGetInt(t, acc, "UI64", false, 44)
+		testGetFloat(t, acc, "F32", false, 44.44)
+		testGetFloat(t, acc, "F64", false, 44.44)
+		testGetBool(t, acc, "B", false, false)
+	}
+}

database/record/base.go

@@ -53,6 +53,14 @@ func (b *Base) SetMeta(meta *Meta) {
 
 // Marshal marshals the object, without the database key or metadata
 func (b *Base) Marshal(format uint8) ([]byte, error) {
+	if b.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if b.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
 	dumped, err := dsd.Dump(b, format)
 	if err != nil {
 		return nil, err

database/record/wrapper.go

@@ -75,6 +75,14 @@ func NewWrapper(key string, meta *Meta, data []byte) (*Wrapper, error) {
 
 // Marshal marshals the object, without the database key or metadata
 func (w *Wrapper) Marshal(storageType uint8) ([]byte, error) {
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if w.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
 	if storageType != dsd.AUTO && storageType != w.Format {
 		return nil, errors.New("could not dump model, wrapped object format mismatch")
 	}

database/record/wrapper_test.go

@@ -19,7 +19,7 @@ func TestWrapper(t *testing.T) {
 	testData := []byte(`J{"a": "b"}`)
 
 	// test wrapper
-	wrapper, err := NewWrapper("test:a", nil, testData)
+	wrapper, err := NewWrapper("test:a", &Meta{}, testData)
 	if err != nil {
 		t.Fatal(err)
 	}