Make dsd formats stronger typed, return parsed format, remove STRING and BYTES format

formats/dsd/compression.go

@@ -4,28 +4,26 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
-	"fmt"
 
 	"github.com/safing/portbase/formats/varint"
 )
 
 // DumpAndCompress stores the interface as a dsd formatted data structure and compresses the resulting data.
-func DumpAndCompress(t interface{}, format uint8, compression uint8) ([]byte, error) {
+func DumpAndCompress(t interface{}, format SerializationFormat, compression CompressionFormat) ([]byte, error) {
+	// Check if compression format is valid.
+	compression, ok := compression.ValidateCompressionFormat()
+	if !ok {
+		return nil, ErrIncompatibleFormat
+	}
+
+	// Dump the given data with the given format.
 	data, err := Dump(t, format)
 	if err != nil {
 		return nil, err
 	}
 
-	// handle special cases
-	switch compression {
-	case NONE:
-		return data, nil
-	case AUTO:
-		compression = GZIP
-	}
-
 	// prepare writer
-	packetFormat := varint.Pack8(compression)
+	packetFormat := varint.Pack8(uint8(compression))
 	buf := bytes.NewBuffer(nil)
 	buf.Write(packetFormat)
 
@@ -53,52 +51,58 @@ func DumpAndCompress(t interface{}, format uint8, compression uint8) ([]byte, er
 			return nil, err
 		}
 	default:
-		return nil, fmt.Errorf("dsd: tried to compress with unknown format %d", format)
+		return nil, ErrIncompatibleFormat
 	}
 
 	return buf.Bytes(), nil
 }
 
 // DecompressAndLoad decompresses the data using the specified compression format and then loads the resulting data blob into the interface.
-func DecompressAndLoad(data []byte, format uint8, t interface{}) (interface{}, error) {
+func DecompressAndLoad(data []byte, compression CompressionFormat, t interface{}) (format SerializationFormat, err error) {
+	// Check if compression format is valid.
+	compression, ok := compression.ValidateCompressionFormat()
+	if !ok {
+		return 0, ErrIncompatibleFormat
+	}
+
 	// prepare reader
 	buf := bytes.NewBuffer(nil)
 
 	// decompress
-	switch format {
+	switch compression {
 	case GZIP:
 		// create gzip reader
 		gzipReader, err := gzip.NewReader(bytes.NewBuffer(data))
 		if err != nil {
-			return nil, err
+			return 0, err
 		}
 
 		// read uncompressed data
 		_, err = buf.ReadFrom(gzipReader)
 		if err != nil {
-			return nil, err
+			return 0, err
 		}
 
 		// flush and verify gzip footer
 		err = gzipReader.Close()
 		if err != nil {
-			return nil, err
+			return 0, err
 		}
 	default:
-		return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
+		return 0, ErrIncompatibleFormat
 	}
 
 	// assign decompressed data
 	data = buf.Bytes()
 
-	// get format
+	formatID, read, err := loadFormat(data)
-	format, read, err := varint.Unpack8(data)
 	if err != nil {
-		return nil, err
+		return 0, err
 	}
-	if len(data) <= read {
+	format, ok = SerializationFormat(formatID).ValidateSerializationFormat()
-		return nil, errNoMoreSpace
+	if !ok {
+		return 0, ErrIncompatibleFormat
 	}
 
-	return LoadAsFormat(data[read:], format, t)
+	return format, LoadAsFormat(data[read:], format, t)
 }

formats/dsd/dsd.go

@@ -9,113 +9,83 @@ import (
 	"fmt"
 
 	"github.com/fxamacker/cbor/v2"
+
 	"github.com/safing/portbase/formats/varint"
 	"github.com/safing/portbase/utils"
 )
 
-// Types.
-const (
-	AUTO = 0
-	NONE = 1
-
-	// Special types.
-	LIST = 76 // L
-
-	// Serialization types.
-	CBOR    = 67 // C
-	GenCode = 71 // G
-	JSON    = 74 // J
-	STRING  = 83 // S
-	BYTES   = 88 // X
-
-	// Compression types.
-	GZIP = 90 // Z
-)
-
-// Errors.
-var (
-	errNoMoreSpace    = errors.New("dsd: no more space left after reading dsd type")
-	errNotImplemented = errors.New("dsd: this type is not yet implemented")
-)
-
 // Load loads an dsd structured data blob into the given interface.
-func Load(data []byte, t interface{}) (interface{}, error) {
+func Load(data []byte, t interface{}) (format SerializationFormat, err error) {
-	format, read, err := varint.Unpack8(data)
+	formatID, read, err := loadFormat(data)
 	if err != nil {
-		return nil, err
+		return 0, err
-	}
-	if len(data) <= read {
-		return nil, errNoMoreSpace
 	}
 
-	switch format {
+	format, ok := SerializationFormat(formatID).ValidateSerializationFormat()
-	case GZIP:
+	if ok {
-		return DecompressAndLoad(data[read:], format, t)
+		return format, LoadAsFormat(data[read:], format, t)
-	default:
-		return LoadAsFormat(data[read:], format, t)
 	}
+	return DecompressAndLoad(data[read:], CompressionFormat(format), t)
 }
 
 // LoadAsFormat loads a data blob into the interface using the specified format.
-func LoadAsFormat(data []byte, format uint8, t interface{}) (interface{}, error) {
+func LoadAsFormat(data []byte, format SerializationFormat, t interface{}) (err error) {
 	switch format {
-	case STRING:
-		return string(data), nil
-	case BYTES:
-		return data, nil
 	case JSON:
-		err := json.Unmarshal(data, t)
+		err = json.Unmarshal(data, t)
 		if err != nil {
-			return nil, fmt.Errorf("dsd: failed to unpack json: %s, data: %s", err, utils.SafeFirst16Bytes(data))
+			return fmt.Errorf("dsd: failed to unpack json: %w, data: %s", err, utils.SafeFirst16Bytes(data))
 		}
-		return t, nil
+		return nil
 	case CBOR:
-		err := cbor.Unmarshal(data, t)
+		err = cbor.Unmarshal(data, t)
 		if err != nil {
-			return nil, fmt.Errorf("dsd: failed to unpack cbor: %s, data: %s", err, utils.SafeFirst16Bytes(data))
+			return fmt.Errorf("dsd: failed to unpack cbor: %w, data: %s", err, utils.SafeFirst16Bytes(data))
 		}
-		return t, nil
+		return nil
 	case GenCode:
 		genCodeStruct, ok := t.(GenCodeCompatible)
 		if !ok {
-			return nil, errors.New("dsd: gencode is not supported by the given data structure")
+			return errors.New("dsd: gencode is not supported by the given data structure")
 		}
-		_, err := genCodeStruct.GenCodeUnmarshal(data)
+		_, err = genCodeStruct.GenCodeUnmarshal(data)
 		if err != nil {
-			return nil, fmt.Errorf("dsd: failed to unpack gencode: %s, data: %s", err, utils.SafeFirst16Bytes(data))
+			return fmt.Errorf("dsd: failed to unpack gencode: %w, data: %s", err, utils.SafeFirst16Bytes(data))
 		}
-		return t, nil
+		return nil
 	default:
-		return nil, fmt.Errorf("dsd: tried to load unknown type %d, data: %s", format, utils.SafeFirst16Bytes(data))
+		return ErrIncompatibleFormat
 	}
 }
 
+func loadFormat(data []byte) (format uint8, read int, err error) {
+	format, read, err = varint.Unpack8(data)
+	if err != nil {
+		return 0, 0, err
+	}
+	if len(data) <= read {
+		return 0, 0, ErrNoMoreSpace
+	}
+
+	return format, read, nil
+}
+
 // Dump stores the interface as a dsd formatted data structure.
-func Dump(t interface{}, format uint8) ([]byte, error) {
+func Dump(t interface{}, format SerializationFormat) ([]byte, error) {
 	return DumpIndent(t, format, "")
 }
 
 // DumpIndent stores the interface as a dsd formatted data structure with indentation, if available.
-func DumpIndent(t interface{}, format uint8, indent string) ([]byte, error) {
+func DumpIndent(t interface{}, format SerializationFormat, indent string) ([]byte, error) {
-	if format == AUTO {
+	format, ok := format.ValidateSerializationFormat()
-		switch t.(type) {
+	if !ok {
-		case string:
+		return nil, ErrIncompatibleFormat
-			format = STRING
-		case []byte:
-			format = BYTES
-		default:
-			format = JSON
-		}
 	}
 
-	f := varint.Pack8(format)
+	f := varint.Pack8(uint8(format))
 	var data []byte
 	var err error
 	switch format {
-	case STRING:
-		data = []byte(t.(string))
-	case BYTES:
-		data = t.([]byte)
 	case JSON:
 		// TODO: use SetEscapeHTML(false)
 		if indent != "" {
@@ -138,12 +108,13 @@ func DumpIndent(t interface{}, format uint8, indent string) ([]byte, error) {
 		}
 		data, err = genCodeStruct.GenCodeMarshal(nil)
 		if err != nil {
-			return nil, fmt.Errorf("dsd: failed to pack gencode struct: %s", err)
+			return nil, fmt.Errorf("dsd: failed to pack gencode struct: %w", err)
 		}
 	default:
-		return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
+		return nil, ErrIncompatibleFormat
 	}
 
-	r := append(f, data...)
+	// TODO: Find a better way to do this.
-	return r, nil
+	f = append(f, data...)
+	return f, nil
 }

formats/dsd/dsd_test.go

@@ -1,8 +1,7 @@
-//nolint:maligned,unparam,gocyclo,gocognit
+//nolint:maligned,gocyclo,gocognit
 package dsd
 
 import (
-	"bytes"
 	"math/big"
 	"reflect"
 	"testing"
@@ -57,137 +56,116 @@ type GenCodeTestStruct struct {
 	Bap  *[]byte
 }
 
+var (
+	simpleSubject = &SimpleTestStruct{
+		"a",
+		0x01,
+	}
+
+	bString      = "b"
+	bBytes  byte = 0x02
+
+	complexSubject = &ComplexTestStruct{
+		-1,
+		-2,
+		-3,
+		-4,
+		-5,
+		1,
+		2,
+		3,
+		4,
+		5,
+		big.NewInt(6),
+		"a",
+		&bString,
+		[]string{"c", "d", "e"},
+		&[]string{"f", "g", "h"},
+		0x01,
+		&bBytes,
+		[]byte{0x03, 0x04, 0x05},
+		&[]byte{0x05, 0x06, 0x07},
+		map[string]string{
+			"a": "b",
+			"c": "d",
+			"e": "f",
+		},
+		&map[string]string{
+			"g": "h",
+			"i": "j",
+			"k": "l",
+		},
+	}
+
+	genCodeSubject = &GenCodeTestStruct{
+		-2,
+		-3,
+		-4,
+		-5,
+		2,
+		3,
+		4,
+		5,
+		"a",
+		&bString,
+		[]string{"c", "d", "e"},
+		&[]string{"f", "g", "h"},
+		0x01,
+		&bBytes,
+		[]byte{0x03, 0x04, 0x05},
+		&[]byte{0x05, 0x06, 0x07},
+	}
+)
+
 func TestConversion(t *testing.T) {
-	compressionFormats := []uint8{NONE, GZIP}
+	t.Parallel()
+
+	compressionFormats := []CompressionFormat{AutoCompress, GZIP}
+	formats := []SerializationFormat{JSON, CBOR}
+
 	for _, compression := range compressionFormats {
-
-		// STRING
-		d, err := DumpAndCompress("abc", STRING, compression)
-		if err != nil {
-			t.Fatalf("Dump error (string): %s", err)
-		}
-
-		s, err := Load(d, nil)
-		if err != nil {
-			t.Fatalf("Load error (string): %s", err)
-		}
-		ts := s.(string)
-
-		if ts != "abc" {
-			t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", ts, "abc")
-		}
-
-		// BYTES
-		d, err = DumpAndCompress([]byte("def"), BYTES, compression)
-		if err != nil {
-			t.Fatalf("Dump error (string): %s", err)
-		}
-
-		b, err := Load(d, nil)
-		if err != nil {
-			t.Fatalf("Load error (string): %s", err)
-		}
-		tb := b.([]byte)
-
-		if !bytes.Equal(tb, []byte("def")) {
-			t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", tb, []byte("def"))
-		}
-
-		// STRUCTS
-		simpleSubject := SimpleTestStruct{
-			"a",
-			0x01,
-		}
-
-		bString := "b"
-		var bBytes byte = 0x02
-
-		complexSubject := ComplexTestStruct{
-			-1,
-			-2,
-			-3,
-			-4,
-			-5,
-			1,
-			2,
-			3,
-			4,
-			5,
-			big.NewInt(6),
-			"a",
-			&bString,
-			[]string{"c", "d", "e"},
-			&[]string{"f", "g", "h"},
-			0x01,
-			&bBytes,
-			[]byte{0x03, 0x04, 0x05},
-			&[]byte{0x05, 0x06, 0x07},
-			map[string]string{
-				"a": "b",
-				"c": "d",
-				"e": "f",
-			},
-			&map[string]string{
-				"g": "h",
-				"i": "j",
-				"k": "l",
-			},
-		}
-
-		genCodeSubject := GenCodeTestStruct{
-			-2,
-			-3,
-			-4,
-			-5,
-			2,
-			3,
-			4,
-			5,
-			"a",
-			&bString,
-			[]string{"c", "d", "e"},
-			&[]string{"f", "g", "h"},
-			0x01,
-			&bBytes,
-			[]byte{0x03, 0x04, 0x05},
-			&[]byte{0x05, 0x06, 0x07},
-		}
-
-		// test all formats (complex)
-		formats := []uint8{JSON, CBOR}
-
 		for _, format := range formats {
 
 			// simple
-			b, err := DumpAndCompress(&simpleSubject, format, compression)
+			var b []byte
+			var err error
+			if compression != AutoCompress {
+				b, err = DumpAndCompress(simpleSubject, format, compression)
+			} else {
+				b, err = Dump(simpleSubject, format)
+			}
 			if err != nil {
 				t.Fatalf("Dump error (simple struct): %s", err)
 			}
 
-			o, err := Load(b, &SimpleTestStruct{})
+			si := &SimpleTestStruct{}
+			_, err = Load(b, si)
 			if err != nil {
 				t.Fatalf("Load error (simple struct): %s", err)
 			}
 
-			if !reflect.DeepEqual(&simpleSubject, o) {
+			if !reflect.DeepEqual(simpleSubject, si) {
 				t.Errorf("Load (simple struct): subject does not match loaded object")
 				t.Errorf("Encoded: %v", string(b))
-				t.Errorf("Compared: %v == %v", &simpleSubject, o)
+				t.Errorf("Compared: %v == %v", simpleSubject, si)
 			}
 
 			// complex
-			b, err = DumpAndCompress(&complexSubject, format, compression)
+			if compression != AutoCompress {
+				b, err = DumpAndCompress(complexSubject, format, compression)
+			} else {
+				b, err = Dump(complexSubject, format)
+			}
 			if err != nil {
 				t.Fatalf("Dump error (complex struct): %s", err)
 			}
 
-			o, err = Load(b, &ComplexTestStruct{})
+			co := &ComplexTestStruct{}
+			_, err = Load(b, co)
 			if err != nil {
 				t.Fatalf("Load error (complex struct): %s", err)
 			}
 
-			co := o.(*ComplexTestStruct)
-
 			if complexSubject.I != co.I {
 				t.Errorf("Load (complex struct): struct.I is not equal (%v != %v)", complexSubject.I, co.I)
 			}
@@ -255,39 +233,46 @@ func TestConversion(t *testing.T) {
 		}
 
 		// test all formats
-		formats = []uint8{JSON, CBOR, GenCode}
+		simplifiedFormatTesting := []SerializationFormat{JSON, CBOR, GenCode}
+
+		for _, format := range simplifiedFormatTesting {
 
-		for _, format := range formats {
 			// simple
-			b, err := DumpAndCompress(&simpleSubject, format, compression)
+			var b []byte
+			var err error
+			if compression != AutoCompress {
+				b, err = DumpAndCompress(simpleSubject, format, compression)
+			} else {
+				b, err = Dump(simpleSubject, format)
+			}
 			if err != nil {
 				t.Fatalf("Dump error (simple struct): %s", err)
 			}
 
-			o, err := Load(b, &SimpleTestStruct{})
+			si := &SimpleTestStruct{}
+			_, err = Load(b, si)
 			if err != nil {
 				t.Fatalf("Load error (simple struct): %s", err)
 			}
 
-			if !reflect.DeepEqual(&simpleSubject, o) {
+			if !reflect.DeepEqual(simpleSubject, si) {
 				t.Errorf("Load (simple struct): subject does not match loaded object")
 				t.Errorf("Encoded: %v", string(b))
-				t.Errorf("Compared: %v == %v", &simpleSubject, o)
+				t.Errorf("Compared: %v == %v", simpleSubject, si)
 			}
 
 			// complex
-			b, err = DumpAndCompress(&genCodeSubject, format, compression)
+			b, err = DumpAndCompress(genCodeSubject, format, compression)
 			if err != nil {
 				t.Fatalf("Dump error (complex struct): %s", err)
 			}
 
-			o, err = Load(b, &GenCodeTestStruct{})
+			co := &GenCodeTestStruct{}
+			_, err = Load(b, co)
 			if err != nil {
 				t.Fatalf("Load error (complex struct): %s", err)
 			}
 
-			co := o.(*GenCodeTestStruct)
-
 			if genCodeSubject.I8 != co.I8 {
 				t.Errorf("Load (complex struct): struct.I8 is not equal (%v != %v)", genCodeSubject.I8, co.I8)
 			}

formats/dsd/format.go

@@ -0,0 +1,71 @@
+package dsd
+
+import "errors"
+
+var (
+	ErrIncompatibleFormat = errors.New("dsd: format is incompatible with operation")
+	ErrNoMoreSpace        = errors.New("dsd: no more space left after reading dsd type")
+	ErrUnknownFormat      = errors.New("dsd: format is unknown")
+)
+
+type SerializationFormat uint8
+
+const (
+	AUTO    SerializationFormat = 0
+	CBOR    SerializationFormat = 67 // C
+	GenCode SerializationFormat = 71 // G
+	JSON    SerializationFormat = 74 // J
+	MsgPack SerializationFormat = 77 // M
+)
+
+type CompressionFormat uint8
+
+const (
+	AutoCompress CompressionFormat = 0
+	GZIP         CompressionFormat = 90 // Z
+)
+
+type SpecialFormat uint8
+
+const (
+	LIST SpecialFormat = 76 // L
+)
+
+var (
+	DefaultSerializationFormat = JSON
+	DefaultCompressionFormat   = GZIP
+)
+
+// ValidateSerializationFormat validates if the format is for serialization,
+// and returns the validated format as well as the result of the validation.
+// If called on the AUTO format, it returns the default serialization format.
+func (format SerializationFormat) ValidateSerializationFormat() (validated SerializationFormat, ok bool) {
+	switch format {
+	case AUTO:
+		return DefaultSerializationFormat, true
+	case CBOR:
+		return format, true
+	case GenCode:
+		return format, true
+	case JSON:
+		return format, true
+	case MsgPack:
+		return format, true
+	default:
+		return 0, false
+	}
+}
+
+// ValidateCompressionFormat validates if the format is for compression,
+// and returns the validated format as well as the result of the validation.
+// If called on the AUTO format, it returns the default compression format.
+func (format CompressionFormat) ValidateCompressionFormat() (validated CompressionFormat, ok bool) {
+	switch format {
+	case AutoCompress:
+		return DefaultCompressionFormat, true
+	case GZIP:
+		return format, true
+	default:
+		return 0, false
+	}
+}

formats/dsd/gencode_test.go

@@ -1,4 +1,4 @@
-//nolint:nakedret,unconvert,gocognit
+//nolint:nakedret,unconvert,gocognit,wastedassign,gofumpt
 package dsd
 
 import (