Add meta data type to record serialization

database/record/base.go

@@ -70,7 +70,7 @@ func (b *Base) SetMeta(meta *Meta) {
 	b.meta = meta
 }
 
-// Marshal marshals the object, without the database key or metadata
+// Marshal marshals the object, without the database key or metadata. It returns nil if the record is deleted.
 func (b *Base) Marshal(self Record, format uint8) ([]byte, error) {
 	if b.Meta() == nil {
 		return nil, errors.New("missing meta")
@@ -96,15 +96,15 @@ func (b *Base) MarshalRecord(self Record) ([]byte, error) {
 	// version
 	c := container.New([]byte{1})
 
-	// meta
+	// meta encoding
-	metaSection, err := b.meta.GenCodeMarshal(nil)
+	metaSection, err := dsd.Dump(b.meta, GenCode)
 	if err != nil {
 		return nil, err
 	}
 	c.AppendAsBlock(metaSection)
 
 	// data
-	dataSection, err := b.Marshal(self, dsd.JSON)
+	dataSection, err := b.Marshal(self, JSON)
 	if err != nil {
 		return nil, err
 	}

database/record/formats.go

@@ -11,5 +11,5 @@ const (
 	BYTES   = dsd.BYTES   // X
 	JSON    = dsd.JSON    // J
 	BSON    = dsd.BSON    // B
-	GenCode = dsd.GenCode // G (reserved)
+	GenCode = dsd.GenCode // G
 )

database/record/meta-bench_test.go

@@ -432,7 +432,7 @@ func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
 func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
 
 	for i := 0; i < b.N; i++ {
-		_, err := dsd.Dump(testMeta, dsd.JSON)
+		_, err := dsd.Dump(testMeta, JSON)
 		if err != nil {
 			b.Errorf("failed to serialize with DSD/JSON: %s", err)
 			return
@@ -444,7 +444,7 @@ func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
 func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
 
 	// Setup
-	encodedData, err := dsd.Dump(testMeta, dsd.JSON)
+	encodedData, err := dsd.Dump(testMeta, JSON)
 	if err != nil {
 		b.Errorf("failed to serialize with DSD/JSON: %s", err)
 		return

database/record/wrapper.go

@@ -37,9 +37,20 @@ func NewRawWrapper(database, key string, data []byte) (*Wrapper, error) {
 	offset += n
 
 	newMeta := &Meta{}
-	_, err = newMeta.GenCodeUnmarshal(metaSection)
+	if len(metaSection) == 34 && metaSection[4] == 0 {
-	if err != nil {
+		// TODO: remove in 2020
-		return nil, fmt.Errorf("could not unmarshal meta section: %s", err)
+		// backward compatibility:
+		// format would byte shift and populate metaSection[4] with value > 0 (would naturally populate >0 at 07.02.2106 07:28:15)
+		// this must be gencode without format
+		_, err = newMeta.GenCodeUnmarshal(metaSection)
+		if err != nil {
+			return nil, fmt.Errorf("could not unmarshal meta section: %s", err)
+		}
+	} else {
+		_, err = dsd.Load(metaSection, newMeta)
+		if err != nil {
+			return nil, fmt.Errorf("could not unmarshal meta section: %s", err)
+		}
 	}
 
 	format, n, err := varint.Unpack8(data[offset:])
@@ -86,7 +97,7 @@ func (w *Wrapper) Marshal(r Record, format uint8) ([]byte, error) {
 		return nil, nil
 	}
 
-	if format != dsd.AUTO && format != w.Format {
+	if format != AUTO && format != w.Format {
 		return nil, errors.New("could not dump model, wrapped object format mismatch")
 	}
 
@@ -109,14 +120,14 @@ func (w *Wrapper) MarshalRecord(r Record) ([]byte, error) {
 	c := container.New([]byte{1})
 
 	// meta
-	metaSection, err := w.meta.GenCodeMarshal(nil)
+	metaSection, err := dsd.Dump(w.meta, GenCode)
 	if err != nil {
 		return nil, err
 	}
 	c.AppendAsBlock(metaSection)
 
 	// data
-	dataSection, err := w.Marshal(r, dsd.JSON)
+	dataSection, err := w.Marshal(r, JSON)
 	if err != nil {
 		return nil, err
 	}
@@ -125,16 +136,6 @@ func (w *Wrapper) MarshalRecord(r Record) ([]byte, error) {
 	return c.CompileData(), nil
 }
 
-// // Lock locks the record.
-// func (w *Wrapper) Lock() {
-// 	w.lock.Lock()
-// }
-//
-// // Unlock unlocks the record.
-// func (w *Wrapper) Unlock() {
-// 	w.lock.Unlock()
-// }
-
 // IsWrapped returns whether the record is a Wrapper.
 func (w *Wrapper) IsWrapped() bool {
 	return true

database/record/wrapper_test.go

@@ -2,9 +2,10 @@ package record
 
 import (
 	"bytes"
+	"errors"
 	"testing"
 
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/container"
 )
 
 func TestWrapper(t *testing.T) {
@@ -24,14 +25,14 @@ func TestWrapper(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if wrapper.Format != dsd.JSON {
+	if wrapper.Format != JSON {
 		t.Error("format mismatch")
 	}
 	if !bytes.Equal(testData, wrapper.Data) {
 		t.Error("data mismatch")
 	}
 
-	encoded, err := wrapper.Marshal(wrapper, dsd.JSON)
+	encoded, err := wrapper.Marshal(wrapper, JSON)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -40,6 +41,7 @@ func TestWrapper(t *testing.T) {
 	}
 
 	wrapper.SetMeta(&Meta{})
+	wrapper.meta.Update()
 	raw, err := wrapper.MarshalRecord(wrapper)
 	if err != nil {
 		t.Fatal(err)
@@ -53,4 +55,42 @@ func TestWrapper(t *testing.T) {
 		t.Error("marshal mismatch")
 	}
 
+	// test new format
+	oldRaw, err := oldWrapperMarshalRecord(wrapper, wrapper)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wrapper3, err := NewRawWrapper("test", "a", oldRaw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(testData, wrapper3.Data) {
+		t.Error("marshal mismatch")
+	}
+}
+
+func oldWrapperMarshalRecord(w *Wrapper, r Record) ([]byte, error) {
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	// version
+	c := container.New([]byte{1})
+
+	// meta
+	metaSection, err := w.meta.GenCodeMarshal(nil)
+	if err != nil {
+		return nil, err
+	}
+	c.AppendAsBlock(metaSection)
+
+	// data
+	dataSection, err := w.Marshal(r, JSON)
+	if err != nil {
+		return nil, err
+	}
+	c.Append(dataSection)
+
+	return c.CompileData(), nil
 }