Add compression and DumpIndent

formats/dsd/compression.go

@@ -0,0 +1,104 @@
+package dsd
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+
+	"github.com/safing/portbase/formats/varint"
+)
+
+// DumpAndCompress stores the interface as a dsd formatted data structure and compresses the resulting data.
+func DumpAndCompress(t interface{}, format uint8, compression uint8) ([]byte, error) {
+	data, err := Dump(t, format)
+	if err != nil {
+		return nil, err
+	}
+
+	// handle special cases
+	switch compression {
+	case NONE:
+		return data, nil
+	case AUTO:
+		compression = GZIP
+	}
+
+	// prepare writer
+	packetFormat := varint.Pack8(compression)
+	buf := bytes.NewBuffer(nil)
+	buf.Write(packetFormat)
+
+	// compress
+	switch compression {
+	case GZIP:
+		// create gzip writer
+		gzipWriter, err := gzip.NewWriterLevel(buf, gzip.BestCompression)
+		if err != nil {
+			return nil, err
+		}
+
+		// write data
+		n, err := gzipWriter.Write(data)
+		if err != nil {
+			return nil, err
+		}
+		if n != len(data) {
+			return nil, errors.New("failed to fully write to gzip compressor")
+		}
+
+		// flush and write gzip footer
+		err = gzipWriter.Close()
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("dsd: tried to compress with unknown format %d", format)
+	}
+
+	return buf.Bytes(), nil
+}
+
+// DecompressAndLoad decompresses the data using the specified compression format and then loads the resulting data blob into the interface.
+func DecompressAndLoad(data []byte, format uint8, t interface{}) (interface{}, error) {
+	// prepare reader
+	buf := bytes.NewBuffer(nil)
+
+	// decompress
+	switch format {
+	case GZIP:
+		// create gzip reader
+		gzipReader, err := gzip.NewReader(bytes.NewBuffer(data))
+		if err != nil {
+			return nil, err
+		}
+
+		// read uncompressed data
+		_, err = buf.ReadFrom(gzipReader)
+		if err != nil {
+			return nil, err
+		}
+
+		// flush and verify gzip footer
+		err = gzipReader.Close()
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
+	}
+
+	// assign decompressed data
+	data = buf.Bytes()
+
+	// get format
+	format, read, err := varint.Unpack8(data)
+	if err != nil {
+		return nil, err
+	}
+	if len(data) <= read {
+		return nil, errNoMoreSpace
+	}
+
+	return LoadAsFormat(data[read:], format, t)
+}

formats/dsd/dsd.go

@@ -16,11 +16,20 @@ import (
 // define types
 const (
 	AUTO = 0
+	NONE = 1
+
+	// special
+	LIST = 76 // L
+
+	// serialization
 	STRING  = 83 // S
 	BYTES   = 88 // X
 	JSON    = 74 // J
 	BSON    = 66 // B
 	GenCode = 71 // G
+
+	// compression
+	GZIP = 90 // Z
 )
 
 // define errors
@@ -29,10 +38,6 @@ var errNotImplemented = errors.New("dsd: this type is not yet implemented")
 
 // Load loads an dsd structured data blob into the given interface.
 func Load(data []byte, t interface{}) (interface{}, error) {
-	if len(data) < 2 {
-		return nil, errNoMoreSpace
-	}
-
 	format, read, err := varint.Unpack8(data)
 	if err != nil {
 		return nil, err
@@ -41,7 +46,12 @@ func Load(data []byte, t interface{}) (interface{}, error) {
 		return nil, errNoMoreSpace
 	}
 
+	switch format {
+	case GZIP:
+		return DecompressAndLoad(data[read:], format, t)
+	default:
 		return LoadAsFormat(data[read:], format, t)
+	}
 }
 
 // LoadAsFormat loads a data blob into the interface using the specified format.
@@ -81,6 +91,11 @@ func LoadAsFormat(data []byte, format uint8, t interface{}) (interface{}, error)
 
 // Dump stores the interface as a dsd formatted data structure.
 func Dump(t interface{}, format uint8) ([]byte, error) {
+	return DumpIndent(t, format, "")
+}
+
+// DumpIndent stores the interface as a dsd formatted data structure with indentation, if available.
+func DumpIndent(t interface{}, format uint8, indent string) ([]byte, error) {
 	if format == AUTO {
 		switch t.(type) {
 		case string:
@@ -102,7 +117,11 @@ func Dump(t interface{}, format uint8) ([]byte, error) {
 		data = t.([]byte)
 	case JSON:
 		// TODO: use SetEscapeHTML(false)
+		if indent != "" {
+			data, err = json.MarshalIndent(t, "", indent)
+		} else {
 			data, err = json.Marshal(t)
+		}
 		if err != nil {
 			return nil, err
 		}
@@ -122,7 +141,7 @@ func Dump(t interface{}, format uint8) ([]byte, error) {
 			return nil, fmt.Errorf("dsd: failed to pack gencode struct: %s", err)
 		}
 	default:
-		return nil, fmt.Errorf("dsd: tried to dump unknown type %d", format)
+		return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
 	}
 
 	r := append(f, data...)

formats/dsd/dsd_test.go

@@ -58,9 +58,11 @@ type GenCodeTestStruct struct {
 }
 
 func TestConversion(t *testing.T) {
+	compressionFormats := []uint8{NONE, GZIP}
+	for _, compression := range compressionFormats {
 
 		// STRING
-	d, err := Dump("abc", STRING)
+		d, err := DumpAndCompress("abc", STRING, compression)
 		if err != nil {
 			t.Fatalf("Dump error (string): %s", err)
 		}
@@ -76,7 +78,7 @@ func TestConversion(t *testing.T) {
 		}
 
 		// BYTES
-	d, err = Dump([]byte("def"), BYTES)
+		d, err = DumpAndCompress([]byte("def"), BYTES, compression)
 		if err != nil {
 			t.Fatalf("Dump error (string): %s", err)
 		}
@@ -156,7 +158,7 @@ func TestConversion(t *testing.T) {
 		for _, format := range formats {
 
 			// simple
-		b, err := Dump(&simpleSubject, format)
+			b, err := DumpAndCompress(&simpleSubject, format, compression)
 			if err != nil {
 				t.Fatalf("Dump error (simple struct): %s", err)
 			}
@@ -173,7 +175,7 @@ func TestConversion(t *testing.T) {
 			}
 
 			// complex
-		b, err = Dump(&complexSubject, format)
+			b, err = DumpAndCompress(&complexSubject, format, compression)
 			if err != nil {
 				t.Fatalf("Dump error (complex struct): %s", err)
 			}
@@ -253,7 +255,7 @@ func TestConversion(t *testing.T) {
 
 		for _, format := range formats {
 			// simple
-		b, err := Dump(&simpleSubject, format)
+			b, err := DumpAndCompress(&simpleSubject, format, compression)
 			if err != nil {
 				t.Fatalf("Dump error (simple struct): %s", err)
 			}
@@ -270,7 +272,7 @@ func TestConversion(t *testing.T) {
 			}
 
 			// complex
-		b, err = Dump(&genCodeSubject, format)
+			b, err = DumpAndCompress(&genCodeSubject, format, compression)
 			if err != nil {
 				t.Fatalf("Dump error (complex struct): %s", err)
 			}
@@ -331,4 +333,6 @@ func TestConversion(t *testing.T) {
 				t.Errorf("Load (complex struct): struct.Bap is not equal (%v != %v)", genCodeSubject.Bap, co.Bap)
 			}
 		}
+
+	}
 }