Merge pull request #22 from safing/feature/container-improvements

General improvements

container/container.go

@@ -2,6 +2,7 @@ package container
 
 import (
 	"errors"
+	"io"
 
 	"github.com/safing/portbase/formats/varint"
 )
@@ -48,12 +49,28 @@ func (c *Container) AppendNumber(n uint64) {
 	c.compartments = append(c.compartments, varint.Pack64(n))
 }
 
+// AppendInt appends an int (varint encoded).
+func (c *Container) AppendInt(n int) {
+	c.compartments = append(c.compartments, varint.Pack64(uint64(n)))
+}
+
 // AppendAsBlock appends the length of the data and the data itself. Data will NOT be copied.
 func (c *Container) AppendAsBlock(data []byte) {
 	c.AppendNumber(uint64(len(data)))
 	c.Append(data)
 }
 
+// AppendContainer appends another Container. Data will NOT be copied.
+func (c *Container) AppendContainer(data *Container) {
+	c.compartments = append(c.compartments, data.compartments...)
+}
+
+// AppendContainerAsBlock appends another Container (length and data). Data will NOT be copied.
+func (c *Container) AppendContainerAsBlock(data *Container) {
+	c.AppendNumber(uint64(data.Length()))
+	c.compartments = append(c.compartments, data.compartments...)
+}
+
 // Length returns the full length of all bytes held by the container.
 func (c *Container) Length() (length int) {
 	for i := c.offset; i < len(c.compartments); i++ {
@@ -92,6 +109,16 @@ func (c *Container) Get(n int) ([]byte, error) {
 	return buf, nil
 }
 
+// GetAsContainer returns the given amount of bytes in a new container. Data will NOT be copied and IS consumed.
+func (c *Container) GetAsContainer(n int) (*Container, error) {
+	new := c.gatherAsContainer(n)
+	if new == nil {
+		return nil, errors.New("container: not enough data to return")
+	}
+	c.skip(n)
+	return new, nil
+}
+
 // GetMax returns as much as possible, but the given amount of bytes at maximum. Data MAY be copied and IS consumed.
 func (c *Container) GetMax(n int) []byte {
 	buf := c.gather(n)
@@ -120,6 +147,21 @@ func (c *Container) WriteToSlice(slice []byte) (n int, containerEmptied bool) {
 	return n, true
 }
 
+// WriteAllTo writes all the data to the given io.Writer. Data IS NOT copied (but may be by writer) and IS NOT consumed.
+func (c *Container) WriteAllTo(writer io.Writer) error {
+	for i := c.offset; i < len(c.compartments); i++ {
+		written := 0
+		for written < len(c.compartments[i]) {
+			n, err := writer.Write(c.compartments[i][written:])
+			if err != nil {
+				return err
+			}
+			written += n
+		}
+	}
+	return nil
+}
+
 func (c *Container) clean() {
 	if c.offset > 100 {
 		c.renewCompartments()
@@ -214,6 +256,23 @@ func (c *Container) gather(n int) []byte {
 	return slice[:n]
 }
 
+func (c *Container) gatherAsContainer(n int) (new *Container) {
+	new = &Container{}
+	for i := c.offset; i < len(c.compartments); i++ {
+		if n >= len(c.compartments[i]) {
+			new.compartments = append(new.compartments, c.compartments[i])
+			n -= len(c.compartments[i])
+		} else {
+			new.compartments = append(new.compartments, c.compartments[i][:n])
+			n = 0
+		}
+	}
+	if n > 0 {
+		return nil
+	}
+	return new
+}
+
 func (c *Container) skip(n int) {
 	for i := c.offset; i < len(c.compartments); i++ {
 		if len(c.compartments[i]) <= n {
@@ -233,7 +292,7 @@ func (c *Container) skip(n int) {
 	c.checkOffset()
 }
 
-// GetNextBlock returns the next block of data defined by a varint (note: data will MAY be copied and IS consumed).
+// GetNextBlock returns the next block of data defined by a varint. Data MAY be copied and IS consumed.
 func (c *Container) GetNextBlock() ([]byte, error) {
 	blockSize, err := c.GetNextN64()
 	if err != nil {
@@ -242,6 +301,15 @@ func (c *Container) GetNextBlock() ([]byte, error) {
 	return c.Get(int(blockSize))
 }
 
+// GetNextBlockAsContainer returns the next block of data as a Container defined by a varint. Data will NOT be copied and IS consumed.
+func (c *Container) GetNextBlockAsContainer() (*Container, error) {
+	blockSize, err := c.GetNextN64()
+	if err != nil {
+		return nil, err
+	}
+	return c.GetAsContainer(int(blockSize))
+}
+
 // GetNextN8 parses and returns a varint of type uint8.
 func (c *Container) GetNextN8() (uint8, error) {
 	buf := c.gather(2)

container/container_test.go

@@ -66,7 +66,12 @@ func TestContainerDataHandling(t *testing.T) {
 	}
 	c8.clean()
 
-	compareMany(t, testData, c1.CompileData(), c2.CompileData(), c3.CompileData(), d4, d5, c6.CompileData(), c7.CompileData(), c8.CompileData())
+	c9 := c8.gatherAsContainer(len(testData))
+
+	c10 := c9.gatherAsContainer(len(testData) - 1)
+	c10.Append(testData[len(testData)-1:])
+
+	compareMany(t, testData, c1.CompileData(), c2.CompileData(), c3.CompileData(), d4, d5, c6.CompileData(), c7.CompileData(), c8.CompileData(), c9.CompileData(), c10.CompileData())
 }
 
 func compareMany(t *testing.T, reference []byte, other ...[]byte) {
@@ -120,6 +125,11 @@ func TestDataFetching(t *testing.T) {
 	if err == nil {
 		t.Error("should fail")
 	}
+
+	_, err = c1.GetAsContainer(1000)
+	if err == nil {
+		t.Error("should fail")
+	}
 }
 
 func TestBlocks(t *testing.T) {

container/doc.go

@@ -0,0 +1,27 @@
+// Package container gives you a []byte slice on steroids, allowing for quick data appending, prepending and fetching as well as transparent error transportation.
+//
+// A Container is basically a [][]byte slice that just appends new []byte slices and only copies things around when necessary.
+//
+// Byte slices added to the Container are not changed or appended, to not corrupt any other data that may be before and after the given slice.
+// If interested, consider the following example to understand why this is important:
+//
+//		package main
+//
+//		import (
+//			"fmt"
+//		)
+//
+//		func main() {
+//			a := []byte{0, 1,2,3,4,5,6,7,8,9}
+//			fmt.Printf("a: %+v\n", a)
+//			fmt.Printf("\nmaking changes...\n(we are not changing a directly)\n\n")
+//			b := a[2:6]
+//			c := append(b, 10, 11)
+//			fmt.Printf("b: %+v\n", b)
+//			fmt.Printf("c: %+v\n", c)
+//			fmt.Printf("a: %+v\n", a)
+//		}
+//
+// run it here: https://play.golang.org/p/xu1BXT3QYeE
+//
+package container

container/serialization.go

@@ -0,0 +1,21 @@
+package container
+
+import (
+	"encoding/json"
+)
+
+// MarshalJSON serializes the container as a JSON byte array.
+func (c *Container) MarshalJSON() ([]byte, error) {
+	return json.Marshal(c.CompileData())
+}
+
+// UnmarshalJSON unserializes a container from a JSON byte array.
+func (c *Container) UnmarshalJSON(data []byte) error {
+	var raw []byte
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+
+	c.compartments = [][]byte{raw}
+	return nil
+}

formats/dsd/compression.go

@@ -0,0 +1,104 @@
+package dsd
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+
+	"github.com/safing/portbase/formats/varint"
+)
+
+// DumpAndCompress stores the interface as a dsd formatted data structure and compresses the resulting data.
+func DumpAndCompress(t interface{}, format uint8, compression uint8) ([]byte, error) {
+	data, err := Dump(t, format)
+	if err != nil {
+		return nil, err
+	}
+
+	// handle special cases
+	switch compression {
+	case NONE:
+		return data, nil
+	case AUTO:
+		compression = GZIP
+	}
+
+	// prepare writer
+	packetFormat := varint.Pack8(compression)
+	buf := bytes.NewBuffer(nil)
+	buf.Write(packetFormat)
+
+	// compress
+	switch compression {
+	case GZIP:
+		// create gzip writer
+		gzipWriter, err := gzip.NewWriterLevel(buf, gzip.BestCompression)
+		if err != nil {
+			return nil, err
+		}
+
+		// write data
+		n, err := gzipWriter.Write(data)
+		if err != nil {
+			return nil, err
+		}
+		if n != len(data) {
+			return nil, errors.New("failed to fully write to gzip compressor")
+		}
+
+		// flush and write gzip footer
+		err = gzipWriter.Close()
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("dsd: tried to compress with unknown format %d", format)
+	}
+
+	return buf.Bytes(), nil
+}
+
+// DecompressAndLoad decompresses the data using the specified compression format and then loads the resulting data blob into the interface.
+func DecompressAndLoad(data []byte, format uint8, t interface{}) (interface{}, error) {
+	// prepare reader
+	buf := bytes.NewBuffer(nil)
+
+	// decompress
+	switch format {
+	case GZIP:
+		// create gzip reader
+		gzipReader, err := gzip.NewReader(bytes.NewBuffer(data))
+		if err != nil {
+			return nil, err
+		}
+
+		// read uncompressed data
+		_, err = buf.ReadFrom(gzipReader)
+		if err != nil {
+			return nil, err
+		}
+
+		// flush and verify gzip footer
+		err = gzipReader.Close()
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
+	}
+
+	// assign decompressed data
+	data = buf.Bytes()
+
+	// get format
+	format, read, err := varint.Unpack8(data)
+	if err != nil {
+		return nil, err
+	}
+	if len(data) <= read {
+		return nil, errNoMoreSpace
+	}
+
+	return LoadAsFormat(data[read:], format, t)
+}

formats/dsd/dsd.go

@@ -15,12 +15,21 @@ import (
 
 // define types
 const (
-	AUTO    = 0
+	AUTO = 0
+	NONE = 1
+
+	// special
+	LIST = 76 // L
+
+	// serialization
 	STRING  = 83 // S
 	BYTES   = 88 // X
 	JSON    = 74 // J
 	BSON    = 66 // B
 	GenCode = 71 // G
+
+	// compression
+	GZIP = 90 // Z
 )
 
 // define errors
@@ -29,10 +38,6 @@ var errNotImplemented = errors.New("dsd: this type is not yet implemented")
 
 // Load loads an dsd structured data blob into the given interface.
 func Load(data []byte, t interface{}) (interface{}, error) {
-	if len(data) < 2 {
-		return nil, errNoMoreSpace
-	}
-
 	format, read, err := varint.Unpack8(data)
 	if err != nil {
 		return nil, err
@@ -41,7 +46,12 @@ func Load(data []byte, t interface{}) (interface{}, error) {
 		return nil, errNoMoreSpace
 	}
 
-	return LoadAsFormat(data[read:], format, t)
+	switch format {
+	case GZIP:
+		return DecompressAndLoad(data[read:], format, t)
+	default:
+		return LoadAsFormat(data[read:], format, t)
+	}
 }
 
 // LoadAsFormat loads a data blob into the interface using the specified format.
@@ -81,6 +91,11 @@ func LoadAsFormat(data []byte, format uint8, t interface{}) (interface{}, error)
 
 // Dump stores the interface as a dsd formatted data structure.
 func Dump(t interface{}, format uint8) ([]byte, error) {
+	return DumpIndent(t, format, "")
+}
+
+// DumpIndent stores the interface as a dsd formatted data structure with indentation, if available.
+func DumpIndent(t interface{}, format uint8, indent string) ([]byte, error) {
 	if format == AUTO {
 		switch t.(type) {
 		case string:
@@ -102,7 +117,11 @@ func Dump(t interface{}, format uint8) ([]byte, error) {
 		data = t.([]byte)
 	case JSON:
 		// TODO: use SetEscapeHTML(false)
-		data, err = json.Marshal(t)
+		if indent != "" {
+			data, err = json.MarshalIndent(t, "", indent)
+		} else {
+			data, err = json.Marshal(t)
+		}
 		if err != nil {
 			return nil, err
 		}
@@ -122,7 +141,7 @@ func Dump(t interface{}, format uint8) ([]byte, error) {
 			return nil, fmt.Errorf("dsd: failed to pack gencode struct: %s", err)
 		}
 	default:
-		return nil, fmt.Errorf("dsd: tried to dump unknown type %d", format)
+		return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
 	}
 
 	r := append(f, data...)

formats/dsd/dsd_test.go

@@ -58,277 +58,281 @@ type GenCodeTestStruct struct {
 }
 
 func TestConversion(t *testing.T) {
+	compressionFormats := []uint8{NONE, GZIP}
+	for _, compression := range compressionFormats {
 
-	// STRING
+		// STRING
-	d, err := Dump("abc", STRING)
+		d, err := DumpAndCompress("abc", STRING, compression)
-	if err != nil {
-		t.Fatalf("Dump error (string): %s", err)
-	}
-
-	s, err := Load(d, nil)
-	if err != nil {
-		t.Fatalf("Load error (string): %s", err)
-	}
-	ts := s.(string)
-
-	if ts != "abc" {
-		t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", ts, "abc")
-	}
-
-	// BYTES
-	d, err = Dump([]byte("def"), BYTES)
-	if err != nil {
-		t.Fatalf("Dump error (string): %s", err)
-	}
-
-	b, err := Load(d, nil)
-	if err != nil {
-		t.Fatalf("Load error (string): %s", err)
-	}
-	tb := b.([]byte)
-
-	if !bytes.Equal(tb, []byte("def")) {
-		t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", tb, []byte("def"))
-	}
-
-	// STRUCTS
-	simpleSubject := SimpleTestStruct{
-		"a",
-		0x01,
-	}
-
-	bString := "b"
-	var bBytes byte = 0x02
-
-	complexSubject := ComplexTestStruct{
-		-1,
-		-2,
-		-3,
-		-4,
-		-5,
-		1,
-		2,
-		3,
-		4,
-		5,
-		"a",
-		&bString,
-		[]string{"c", "d", "e"},
-		&[]string{"f", "g", "h"},
-		0x01,
-		&bBytes,
-		[]byte{0x03, 0x04, 0x05},
-		&[]byte{0x05, 0x06, 0x07},
-		map[string]string{
-			"a": "b",
-			"c": "d",
-			"e": "f",
-		},
-		&map[string]string{
-			"g": "h",
-			"i": "j",
-			"k": "l",
-		},
-	}
-
-	genCodeSubject := GenCodeTestStruct{
-		-2,
-		-3,
-		-4,
-		-5,
-		2,
-		3,
-		4,
-		5,
-		"a",
-		&bString,
-		[]string{"c", "d", "e"},
-		&[]string{"f", "g", "h"},
-		0x01,
-		&bBytes,
-		[]byte{0x03, 0x04, 0x05},
-		&[]byte{0x05, 0x06, 0x07},
-	}
-
-	// test all formats (complex)
-	formats := []uint8{JSON}
-
-	for _, format := range formats {
-
-		// simple
-		b, err := Dump(&simpleSubject, format)
 		if err != nil {
-			t.Fatalf("Dump error (simple struct): %s", err)
+			t.Fatalf("Dump error (string): %s", err)
 		}
 
-		o, err := Load(b, &SimpleTestStruct{})
+		s, err := Load(d, nil)
 		if err != nil {
-			t.Fatalf("Load error (simple struct): %s", err)
+			t.Fatalf("Load error (string): %s", err)
+		}
+		ts := s.(string)
+
+		if ts != "abc" {
+			t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", ts, "abc")
 		}
 
-		if !reflect.DeepEqual(&simpleSubject, o) {
+		// BYTES
-			t.Errorf("Load (simple struct): subject does not match loaded object")
+		d, err = DumpAndCompress([]byte("def"), BYTES, compression)
-			t.Errorf("Encoded: %v", string(b))
-			t.Errorf("Compared: %v == %v", &simpleSubject, o)
-		}
-
-		// complex
-		b, err = Dump(&complexSubject, format)
 		if err != nil {
-			t.Fatalf("Dump error (complex struct): %s", err)
+			t.Fatalf("Dump error (string): %s", err)
 		}
 
-		o, err = Load(b, &ComplexTestStruct{})
+		b, err := Load(d, nil)
 		if err != nil {
-			t.Fatalf("Load error (complex struct): %s", err)
+			t.Fatalf("Load error (string): %s", err)
+		}
+		tb := b.([]byte)
+
+		if !bytes.Equal(tb, []byte("def")) {
+			t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", tb, []byte("def"))
 		}
 
-		co := o.(*ComplexTestStruct)
+		// STRUCTS
-
+		simpleSubject := SimpleTestStruct{
-		if complexSubject.I != co.I {
+			"a",
-			t.Errorf("Load (complex struct): struct.I is not equal (%v != %v)", complexSubject.I, co.I)
+			0x01,
-		}
-		if complexSubject.I8 != co.I8 {
-			t.Errorf("Load (complex struct): struct.I8 is not equal (%v != %v)", complexSubject.I8, co.I8)
-		}
-		if complexSubject.I16 != co.I16 {
-			t.Errorf("Load (complex struct): struct.I16 is not equal (%v != %v)", complexSubject.I16, co.I16)
-		}
-		if complexSubject.I32 != co.I32 {
-			t.Errorf("Load (complex struct): struct.I32 is not equal (%v != %v)", complexSubject.I32, co.I32)
-		}
-		if complexSubject.I64 != co.I64 {
-			t.Errorf("Load (complex struct): struct.I64 is not equal (%v != %v)", complexSubject.I64, co.I64)
-		}
-		if complexSubject.UI != co.UI {
-			t.Errorf("Load (complex struct): struct.UI is not equal (%v != %v)", complexSubject.UI, co.UI)
-		}
-		if complexSubject.UI8 != co.UI8 {
-			t.Errorf("Load (complex struct): struct.UI8 is not equal (%v != %v)", complexSubject.UI8, co.UI8)
-		}
-		if complexSubject.UI16 != co.UI16 {
-			t.Errorf("Load (complex struct): struct.UI16 is not equal (%v != %v)", complexSubject.UI16, co.UI16)
-		}
-		if complexSubject.UI32 != co.UI32 {
-			t.Errorf("Load (complex struct): struct.UI32 is not equal (%v != %v)", complexSubject.UI32, co.UI32)
-		}
-		if complexSubject.UI64 != co.UI64 {
-			t.Errorf("Load (complex struct): struct.UI64 is not equal (%v != %v)", complexSubject.UI64, co.UI64)
-		}
-		if complexSubject.S != co.S {
-			t.Errorf("Load (complex struct): struct.S is not equal (%v != %v)", complexSubject.S, co.S)
-		}
-		if !reflect.DeepEqual(complexSubject.Sp, co.Sp) {
-			t.Errorf("Load (complex struct): struct.Sp is not equal (%v != %v)", complexSubject.Sp, co.Sp)
-		}
-		if !reflect.DeepEqual(complexSubject.Sa, co.Sa) {
-			t.Errorf("Load (complex struct): struct.Sa is not equal (%v != %v)", complexSubject.Sa, co.Sa)
-		}
-		if !reflect.DeepEqual(complexSubject.Sap, co.Sap) {
-			t.Errorf("Load (complex struct): struct.Sap is not equal (%v != %v)", complexSubject.Sap, co.Sap)
-		}
-		if complexSubject.B != co.B {
-			t.Errorf("Load (complex struct): struct.B is not equal (%v != %v)", complexSubject.B, co.B)
-		}
-		if !reflect.DeepEqual(complexSubject.Bp, co.Bp) {
-			t.Errorf("Load (complex struct): struct.Bp is not equal (%v != %v)", complexSubject.Bp, co.Bp)
-		}
-		if !reflect.DeepEqual(complexSubject.Ba, co.Ba) {
-			t.Errorf("Load (complex struct): struct.Ba is not equal (%v != %v)", complexSubject.Ba, co.Ba)
-		}
-		if !reflect.DeepEqual(complexSubject.Bap, co.Bap) {
-			t.Errorf("Load (complex struct): struct.Bap is not equal (%v != %v)", complexSubject.Bap, co.Bap)
-		}
-		if !reflect.DeepEqual(complexSubject.M, co.M) {
-			t.Errorf("Load (complex struct): struct.M is not equal (%v != %v)", complexSubject.M, co.M)
-		}
-		if !reflect.DeepEqual(complexSubject.Mp, co.Mp) {
-			t.Errorf("Load (complex struct): struct.Mp is not equal (%v != %v)", complexSubject.Mp, co.Mp)
 		}
 
-	}
+		bString := "b"
+		var bBytes byte = 0x02
 
-	// test all formats
+		complexSubject := ComplexTestStruct{
-	formats = []uint8{JSON, GenCode}
+			-1,
-
+			-2,
-	for _, format := range formats {
+			-3,
-		// simple
+			-4,
-		b, err := Dump(&simpleSubject, format)
+			-5,
-		if err != nil {
+			1,
-			t.Fatalf("Dump error (simple struct): %s", err)
+			2,
+			3,
+			4,
+			5,
+			"a",
+			&bString,
+			[]string{"c", "d", "e"},
+			&[]string{"f", "g", "h"},
+			0x01,
+			&bBytes,
+			[]byte{0x03, 0x04, 0x05},
+			&[]byte{0x05, 0x06, 0x07},
+			map[string]string{
+				"a": "b",
+				"c": "d",
+				"e": "f",
+			},
+			&map[string]string{
+				"g": "h",
+				"i": "j",
+				"k": "l",
+			},
 		}
 
-		o, err := Load(b, &SimpleTestStruct{})
+		genCodeSubject := GenCodeTestStruct{
-		if err != nil {
+			-2,
-			t.Fatalf("Load error (simple struct): %s", err)
+			-3,
+			-4,
+			-5,
+			2,
+			3,
+			4,
+			5,
+			"a",
+			&bString,
+			[]string{"c", "d", "e"},
+			&[]string{"f", "g", "h"},
+			0x01,
+			&bBytes,
+			[]byte{0x03, 0x04, 0x05},
+			&[]byte{0x05, 0x06, 0x07},
 		}
 
-		if !reflect.DeepEqual(&simpleSubject, o) {
+		// test all formats (complex)
-			t.Errorf("Load (simple struct): subject does not match loaded object")
+		formats := []uint8{JSON}
-			t.Errorf("Encoded: %v", string(b))
+
-			t.Errorf("Compared: %v == %v", &simpleSubject, o)
+		for _, format := range formats {
+
+			// simple
+			b, err := DumpAndCompress(&simpleSubject, format, compression)
+			if err != nil {
+				t.Fatalf("Dump error (simple struct): %s", err)
+			}
+
+			o, err := Load(b, &SimpleTestStruct{})
+			if err != nil {
+				t.Fatalf("Load error (simple struct): %s", err)
+			}
+
+			if !reflect.DeepEqual(&simpleSubject, o) {
+				t.Errorf("Load (simple struct): subject does not match loaded object")
+				t.Errorf("Encoded: %v", string(b))
+				t.Errorf("Compared: %v == %v", &simpleSubject, o)
+			}
+
+			// complex
+			b, err = DumpAndCompress(&complexSubject, format, compression)
+			if err != nil {
+				t.Fatalf("Dump error (complex struct): %s", err)
+			}
+
+			o, err = Load(b, &ComplexTestStruct{})
+			if err != nil {
+				t.Fatalf("Load error (complex struct): %s", err)
+			}
+
+			co := o.(*ComplexTestStruct)
+
+			if complexSubject.I != co.I {
+				t.Errorf("Load (complex struct): struct.I is not equal (%v != %v)", complexSubject.I, co.I)
+			}
+			if complexSubject.I8 != co.I8 {
+				t.Errorf("Load (complex struct): struct.I8 is not equal (%v != %v)", complexSubject.I8, co.I8)
+			}
+			if complexSubject.I16 != co.I16 {
+				t.Errorf("Load (complex struct): struct.I16 is not equal (%v != %v)", complexSubject.I16, co.I16)
+			}
+			if complexSubject.I32 != co.I32 {
+				t.Errorf("Load (complex struct): struct.I32 is not equal (%v != %v)", complexSubject.I32, co.I32)
+			}
+			if complexSubject.I64 != co.I64 {
+				t.Errorf("Load (complex struct): struct.I64 is not equal (%v != %v)", complexSubject.I64, co.I64)
+			}
+			if complexSubject.UI != co.UI {
+				t.Errorf("Load (complex struct): struct.UI is not equal (%v != %v)", complexSubject.UI, co.UI)
+			}
+			if complexSubject.UI8 != co.UI8 {
+				t.Errorf("Load (complex struct): struct.UI8 is not equal (%v != %v)", complexSubject.UI8, co.UI8)
+			}
+			if complexSubject.UI16 != co.UI16 {
+				t.Errorf("Load (complex struct): struct.UI16 is not equal (%v != %v)", complexSubject.UI16, co.UI16)
+			}
+			if complexSubject.UI32 != co.UI32 {
+				t.Errorf("Load (complex struct): struct.UI32 is not equal (%v != %v)", complexSubject.UI32, co.UI32)
+			}
+			if complexSubject.UI64 != co.UI64 {
+				t.Errorf("Load (complex struct): struct.UI64 is not equal (%v != %v)", complexSubject.UI64, co.UI64)
+			}
+			if complexSubject.S != co.S {
+				t.Errorf("Load (complex struct): struct.S is not equal (%v != %v)", complexSubject.S, co.S)
+			}
+			if !reflect.DeepEqual(complexSubject.Sp, co.Sp) {
+				t.Errorf("Load (complex struct): struct.Sp is not equal (%v != %v)", complexSubject.Sp, co.Sp)
+			}
+			if !reflect.DeepEqual(complexSubject.Sa, co.Sa) {
+				t.Errorf("Load (complex struct): struct.Sa is not equal (%v != %v)", complexSubject.Sa, co.Sa)
+			}
+			if !reflect.DeepEqual(complexSubject.Sap, co.Sap) {
+				t.Errorf("Load (complex struct): struct.Sap is not equal (%v != %v)", complexSubject.Sap, co.Sap)
+			}
+			if complexSubject.B != co.B {
+				t.Errorf("Load (complex struct): struct.B is not equal (%v != %v)", complexSubject.B, co.B)
+			}
+			if !reflect.DeepEqual(complexSubject.Bp, co.Bp) {
+				t.Errorf("Load (complex struct): struct.Bp is not equal (%v != %v)", complexSubject.Bp, co.Bp)
+			}
+			if !reflect.DeepEqual(complexSubject.Ba, co.Ba) {
+				t.Errorf("Load (complex struct): struct.Ba is not equal (%v != %v)", complexSubject.Ba, co.Ba)
+			}
+			if !reflect.DeepEqual(complexSubject.Bap, co.Bap) {
+				t.Errorf("Load (complex struct): struct.Bap is not equal (%v != %v)", complexSubject.Bap, co.Bap)
+			}
+			if !reflect.DeepEqual(complexSubject.M, co.M) {
+				t.Errorf("Load (complex struct): struct.M is not equal (%v != %v)", complexSubject.M, co.M)
+			}
+			if !reflect.DeepEqual(complexSubject.Mp, co.Mp) {
+				t.Errorf("Load (complex struct): struct.Mp is not equal (%v != %v)", complexSubject.Mp, co.Mp)
+			}
+
 		}
 
-		// complex
+		// test all formats
-		b, err = Dump(&genCodeSubject, format)
+		formats = []uint8{JSON, GenCode}
-		if err != nil {
+
-			t.Fatalf("Dump error (complex struct): %s", err)
+		for _, format := range formats {
+			// simple
+			b, err := DumpAndCompress(&simpleSubject, format, compression)
+			if err != nil {
+				t.Fatalf("Dump error (simple struct): %s", err)
+			}
+
+			o, err := Load(b, &SimpleTestStruct{})
+			if err != nil {
+				t.Fatalf("Load error (simple struct): %s", err)
+			}
+
+			if !reflect.DeepEqual(&simpleSubject, o) {
+				t.Errorf("Load (simple struct): subject does not match loaded object")
+				t.Errorf("Encoded: %v", string(b))
+				t.Errorf("Compared: %v == %v", &simpleSubject, o)
+			}
+
+			// complex
+			b, err = DumpAndCompress(&genCodeSubject, format, compression)
+			if err != nil {
+				t.Fatalf("Dump error (complex struct): %s", err)
+			}
+
+			o, err = Load(b, &GenCodeTestStruct{})
+			if err != nil {
+				t.Fatalf("Load error (complex struct): %s", err)
+			}
+
+			co := o.(*GenCodeTestStruct)
+
+			if genCodeSubject.I8 != co.I8 {
+				t.Errorf("Load (complex struct): struct.I8 is not equal (%v != %v)", genCodeSubject.I8, co.I8)
+			}
+			if genCodeSubject.I16 != co.I16 {
+				t.Errorf("Load (complex struct): struct.I16 is not equal (%v != %v)", genCodeSubject.I16, co.I16)
+			}
+			if genCodeSubject.I32 != co.I32 {
+				t.Errorf("Load (complex struct): struct.I32 is not equal (%v != %v)", genCodeSubject.I32, co.I32)
+			}
+			if genCodeSubject.I64 != co.I64 {
+				t.Errorf("Load (complex struct): struct.I64 is not equal (%v != %v)", genCodeSubject.I64, co.I64)
+			}
+			if genCodeSubject.UI8 != co.UI8 {
+				t.Errorf("Load (complex struct): struct.UI8 is not equal (%v != %v)", genCodeSubject.UI8, co.UI8)
+			}
+			if genCodeSubject.UI16 != co.UI16 {
+				t.Errorf("Load (complex struct): struct.UI16 is not equal (%v != %v)", genCodeSubject.UI16, co.UI16)
+			}
+			if genCodeSubject.UI32 != co.UI32 {
+				t.Errorf("Load (complex struct): struct.UI32 is not equal (%v != %v)", genCodeSubject.UI32, co.UI32)
+			}
+			if genCodeSubject.UI64 != co.UI64 {
+				t.Errorf("Load (complex struct): struct.UI64 is not equal (%v != %v)", genCodeSubject.UI64, co.UI64)
+			}
+			if genCodeSubject.S != co.S {
+				t.Errorf("Load (complex struct): struct.S is not equal (%v != %v)", genCodeSubject.S, co.S)
+			}
+			if !reflect.DeepEqual(genCodeSubject.Sp, co.Sp) {
+				t.Errorf("Load (complex struct): struct.Sp is not equal (%v != %v)", genCodeSubject.Sp, co.Sp)
+			}
+			if !reflect.DeepEqual(genCodeSubject.Sa, co.Sa) {
+				t.Errorf("Load (complex struct): struct.Sa is not equal (%v != %v)", genCodeSubject.Sa, co.Sa)
+			}
+			if !reflect.DeepEqual(genCodeSubject.Sap, co.Sap) {
+				t.Errorf("Load (complex struct): struct.Sap is not equal (%v != %v)", genCodeSubject.Sap, co.Sap)
+			}
+			if genCodeSubject.B != co.B {
+				t.Errorf("Load (complex struct): struct.B is not equal (%v != %v)", genCodeSubject.B, co.B)
+			}
+			if !reflect.DeepEqual(genCodeSubject.Bp, co.Bp) {
+				t.Errorf("Load (complex struct): struct.Bp is not equal (%v != %v)", genCodeSubject.Bp, co.Bp)
+			}
+			if !reflect.DeepEqual(genCodeSubject.Ba, co.Ba) {
+				t.Errorf("Load (complex struct): struct.Ba is not equal (%v != %v)", genCodeSubject.Ba, co.Ba)
+			}
+			if !reflect.DeepEqual(genCodeSubject.Bap, co.Bap) {
+				t.Errorf("Load (complex struct): struct.Bap is not equal (%v != %v)", genCodeSubject.Bap, co.Bap)
+			}
 		}
 
-		o, err = Load(b, &GenCodeTestStruct{})
-		if err != nil {
-			t.Fatalf("Load error (complex struct): %s", err)
-		}
-
-		co := o.(*GenCodeTestStruct)
-
-		if genCodeSubject.I8 != co.I8 {
-			t.Errorf("Load (complex struct): struct.I8 is not equal (%v != %v)", genCodeSubject.I8, co.I8)
-		}
-		if genCodeSubject.I16 != co.I16 {
-			t.Errorf("Load (complex struct): struct.I16 is not equal (%v != %v)", genCodeSubject.I16, co.I16)
-		}
-		if genCodeSubject.I32 != co.I32 {
-			t.Errorf("Load (complex struct): struct.I32 is not equal (%v != %v)", genCodeSubject.I32, co.I32)
-		}
-		if genCodeSubject.I64 != co.I64 {
-			t.Errorf("Load (complex struct): struct.I64 is not equal (%v != %v)", genCodeSubject.I64, co.I64)
-		}
-		if genCodeSubject.UI8 != co.UI8 {
-			t.Errorf("Load (complex struct): struct.UI8 is not equal (%v != %v)", genCodeSubject.UI8, co.UI8)
-		}
-		if genCodeSubject.UI16 != co.UI16 {
-			t.Errorf("Load (complex struct): struct.UI16 is not equal (%v != %v)", genCodeSubject.UI16, co.UI16)
-		}
-		if genCodeSubject.UI32 != co.UI32 {
-			t.Errorf("Load (complex struct): struct.UI32 is not equal (%v != %v)", genCodeSubject.UI32, co.UI32)
-		}
-		if genCodeSubject.UI64 != co.UI64 {
-			t.Errorf("Load (complex struct): struct.UI64 is not equal (%v != %v)", genCodeSubject.UI64, co.UI64)
-		}
-		if genCodeSubject.S != co.S {
-			t.Errorf("Load (complex struct): struct.S is not equal (%v != %v)", genCodeSubject.S, co.S)
-		}
-		if !reflect.DeepEqual(genCodeSubject.Sp, co.Sp) {
-			t.Errorf("Load (complex struct): struct.Sp is not equal (%v != %v)", genCodeSubject.Sp, co.Sp)
-		}
-		if !reflect.DeepEqual(genCodeSubject.Sa, co.Sa) {
-			t.Errorf("Load (complex struct): struct.Sa is not equal (%v != %v)", genCodeSubject.Sa, co.Sa)
-		}
-		if !reflect.DeepEqual(genCodeSubject.Sap, co.Sap) {
-			t.Errorf("Load (complex struct): struct.Sap is not equal (%v != %v)", genCodeSubject.Sap, co.Sap)
-		}
-		if genCodeSubject.B != co.B {
-			t.Errorf("Load (complex struct): struct.B is not equal (%v != %v)", genCodeSubject.B, co.B)
-		}
-		if !reflect.DeepEqual(genCodeSubject.Bp, co.Bp) {
-			t.Errorf("Load (complex struct): struct.Bp is not equal (%v != %v)", genCodeSubject.Bp, co.Bp)
-		}
-		if !reflect.DeepEqual(genCodeSubject.Ba, co.Ba) {
-			t.Errorf("Load (complex struct): struct.Ba is not equal (%v != %v)", genCodeSubject.Ba, co.Ba)
-		}
-		if !reflect.DeepEqual(genCodeSubject.Bap, co.Bap) {
-			t.Errorf("Load (complex struct): struct.Bap is not equal (%v != %v)", genCodeSubject.Bap, co.Bap)
-		}
 	}
 }

info/flags.go

@@ -1,62 +0,0 @@
-package info
-
-import (
-	"errors"
-	"flag"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/safing/portbase/modules"
-)
-
-var (
-	showVersion bool
-)
-
-func init() {
-	modules.Register("info", prep, nil, nil)
-
-	flag.BoolVar(&showVersion, "version", false, "show version and exit")
-}
-
-func prep() error {
-	err := CheckVersion()
-	if err != nil {
-		return err
-	}
-
-	if PrintVersion() {
-		return modules.ErrCleanExit
-	}
-	return nil
-}
-
-// CheckVersion checks if the metadata is ok.
-func CheckVersion() error {
-	if !strings.HasSuffix(os.Args[0], ".test") {
-		if name == "[NAME]" {
-			return errors.New("must call SetInfo() before calling CheckVersion()")
-		}
-		if version == "[version unknown]" ||
-			commit == "[commit unknown]" ||
-			license == "[license unknown]" ||
-			buildOptions == "[options unknown]" ||
-			buildUser == "[user unknown]" ||
-			buildHost == "[host unknown]" ||
-			buildDate == "[date unknown]" ||
-			buildSource == "[source unknown]" {
-			return errors.New("please build using the supplied build script.\n$ ./build {main.go|...}")
-		}
-	}
-	return nil
-}
-
-// PrintVersion prints the version, if requested, and returns if it did so.
-func PrintVersion() (printed bool) {
-	if showVersion {
-		fmt.Println(FullVersion())
-		return true
-	}
-	return false
-}

info/module/flags.go

@@ -0,0 +1,40 @@
+package module
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/safing/portbase/info"
+	"github.com/safing/portbase/modules"
+)
+
+var (
+	showVersion bool
+)
+
+func init() {
+	modules.Register("info", prep, nil, nil)
+
+	flag.BoolVar(&showVersion, "version", false, "show version and exit")
+}
+
+func prep() error {
+	err := info.CheckVersion()
+	if err != nil {
+		return err
+	}
+
+	if printVersion() {
+		return modules.ErrCleanExit
+	}
+	return nil
+}
+
+// printVersion prints the version, if requested, and returns if it did so.
+func printVersion() (printed bool) {
+	if showVersion {
+		fmt.Println(info.FullVersion())
+		return true
+	}
+	return false
+}

info/version.go

@@ -1,7 +1,9 @@
 package info
 
 import (
+	"errors"
 	"fmt"
+	"os"
 	"runtime"
 	"strings"
 )
@@ -80,3 +82,23 @@ func FullVersion() string {
 	s += fmt.Sprintf("\nLicensed under the %s license.\nThe source code is available here: %s", license, buildSource)
 	return s
 }
+
+// CheckVersion checks if the metadata is ok.
+func CheckVersion() error {
+	if !strings.HasSuffix(os.Args[0], ".test") {
+		if name == "[NAME]" {
+			return errors.New("must call SetInfo() before calling CheckVersion()")
+		}
+		if version == "[version unknown]" ||
+			commit == "[commit unknown]" ||
+			license == "[license unknown]" ||
+			buildOptions == "[options unknown]" ||
+			buildUser == "[user unknown]" ||
+			buildHost == "[host unknown]" ||
+			buildDate == "[date unknown]" ||
+			buildSource == "[source unknown]" {
+			return errors.New("please build using the supplied build script.\n$ ./build {main.go|...}")
+		}
+	}
+	return nil
+}