vikarti.anatra/safing-portbase
1
0
Fork 0
mirror of https://github.com/safing/portbase synced 2025-04-17 07:59:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request from safing/feature/dsd-improvements

Browse source 
Improve DSD and API
This commit is contained in:
Patrick Pacher 2021-11-26 08:55:04 +01:00 • committed by GitHub
commit b3dd9a1b3f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 574 additions and 276 deletions

11
api/database.go
View file

@ -11,6 +11,7 @@ import (
"github.com/tidwall/sjson"
"github.com/safing/portbase/database/iterator"
"github.com/safing/portbase/formats/dsd"
"github.com/safing/portbase/formats/varint"
"github.com/gorilla/websocket"
@ -534,9 +535,9 @@ func (api *DatabaseAPI) handlePut(opID []byte, key string, data []byte, create b
}
// TODO - staged for deletion: remove transition code
// if data[0] != record.JSON {
// if data[0] != dsd.JSON {
// typedData := make([]byte, len(data)+1)
// typedData[0] = record.JSON
// typedData[0] = dsd.JSON
// copy(typedData[1:], data)
// data = typedData
// }
@ -631,13 +632,13 @@ func marshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) {
defer r.Unlock()
// Pour record into JSON.
jsonData, err := r.Marshal(r, record.JSON)
jsonData, err := r.Marshal(r, dsd.JSON)
if err != nil {
return nil, err
}
// Remove JSON identifier for manual editing.
jsonData = bytes.TrimPrefix(jsonData, varint.Pack8(record.JSON))
jsonData = bytes.TrimPrefix(jsonData, varint.Pack8(dsd.JSON))
// Add metadata.
jsonData, err = sjson.SetBytes(jsonData, "_meta", r.Meta())
@ -653,7 +654,7 @@ func marshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) {
// Add JSON identifier again.
if withDSDIdentifier {
formatID := varint.Pack8(record.JSON)
formatID := varint.Pack8(dsd.JSON)
finalData := make([]byte, 0, len(formatID)+len(jsonData))
finalData = append(finalData, formatID...)
finalData = append(finalData, jsonData...)

142
api/endpoints.go
View file

@ -22,11 +22,41 @@ import (
// Path and at least one permission are required.
// As is exactly one function.
type Endpoint struct {
Path string
MimeType string
Read Permission
Write Permission
BelongsTo *modules.Module
// Path describes the URL path of the endpoint.
Path string
// MimeType defines the content type of the returned data.
MimeType string
// Read defines the required read permission.
Read Permission `json:",omitempty"`
// ReadMethod sets the required read method for the endpoint.
// Available methods are:
// GET: Returns data only, no action is taken, nothing is changed.
// If omitted, defaults to GET.
//
// This field is currently being introduced and will only warn and not deny
// access if the write method does not match.
ReadMethod string `json:",omitempty"`
// Write defines the required write permission.
Write Permission `json:",omitempty"`
// WriteMethod sets the required write method for the endpoint.
// Available methods are:
// POST: Create a new resource; Change a status; Execute a function
// PUT: Update an existing resource
// DELETE: Remove an existing resource
// If omitted, defaults to POST.
//
// This field is currently being introduced and will only warn and not deny
// access if the write method does not match.
WriteMethod string `json:",omitempty"`
// BelongsTo defines which module this endpoint belongs to.
// The endpoint will not be accessible if the module is not online.
BelongsTo *modules.Module `json:"-"`
// ActionFunc is for simple actions with a return message for the user.
ActionFunc ActionFunc `json:"-"`
@ -48,7 +78,7 @@ type Endpoint struct {
Name string
Description string
Parameters []Parameter
Parameters []Parameter `json:",omitempty"`
}
// Parameter describes a parameterized variation of an endpoint.
@ -59,6 +89,41 @@ type Parameter struct {
Description string
}
// HTTPStatusProvider is an interface for errors to provide a custom HTTP
// status code.
type HTTPStatusProvider interface {
HTTPStatus() int
}
// HTTPStatusError represents an error with an HTTP status code.
type HTTPStatusError struct {
err error
code int
}
// Error returns the error message.
func (e *HTTPStatusError) Error() string {
return e.err.Error()
}
// Unwrap return the wrapped error.
func (e *HTTPStatusError) Unwrap() error {
return e.err
}
// HTTPStatus returns the HTTP status code this error.
func (e *HTTPStatusError) HTTPStatus() int {
return e.code
}
// ErrorWithStatus adds the HTTP status code to the error.
func ErrorWithStatus(err error, code int) error {
return &HTTPStatusError{
err: err,
code: code,
}
}
type (
// ActionFunc is for simple actions with a return message for the user.
ActionFunc func(ar *Request) (msg string, err error)
@ -172,6 +237,36 @@ func (e *Endpoint) check() error {
return errors.New("invalid write permission")
}
// Check methods.
if e.Read != NotSupported {
switch e.ReadMethod {
case http.MethodGet:
// All good.
case "":
// Set to default.
e.ReadMethod = http.MethodGet
default:
return errors.New("invalid read method")
}
} else {
e.ReadMethod = ""
}
if e.Write != NotSupported {
switch e.WriteMethod {
case http.MethodPost,
http.MethodPut,
http.MethodDelete:
// All good.
case "":
// Set to default.
e.WriteMethod = http.MethodPost
default:
return errors.New("invalid write method")
}
} else {
e.WriteMethod = ""
}
// Check functions.
var defaultMimeType string
fnCnt := 0
@ -276,6 +371,27 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// TODO: Return errors instead of warnings, also update the field docs.
if isReadMethod(r.Method) {
if r.Method != e.ReadMethod {
log.Tracer(r.Context()).Warningf(
"api: method %q does not match required read method %q%s",
" - this will be an error and abort the request in the future",
r.Method,
e.ReadMethod,
)
}
} else {
if r.Method != e.WriteMethod {
log.Tracer(r.Context()).Warningf(
"api: method %q does not match required write method %q%s",
" - this will be an error and abort the request in the future",
r.Method,
e.ReadMethod,
)
}
}
switch r.Method {
case http.MethodHead:
w.WriteHeader(http.StatusOK)
@ -340,7 +456,19 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Check for handler error.
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// if statusProvider, ok := err.(HTTPStatusProvider); ok {
var statusProvider HTTPStatusProvider
if errors.As(err, &statusProvider) {
http.Error(w, err.Error(), statusProvider.HTTPStatus())
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Check if there is no response data.
if len(responseData) == 0 {
w.WriteHeader(http.StatusNoContent)
return
}

1
api/endpoints_meta.go
View file

@ -51,7 +51,6 @@ func registerMetaEndpoints() error {
if err := RegisterEndpoint(Endpoint{
Path: "auth/reset",
Read: PermitAnyone,
Write: PermitAnyone,
HandlerFunc: authReset,
Name: "Reset Authenticated Session",
Description: "Resets authentication status internally and in the browser.",

27
api/router.go
View file

@ -4,11 +4,14 @@ import (
"context"
"errors"
"net/http"
"net/url"
"path"
"strings"
"sync"
"time"
"github.com/safing/portbase/utils"
"github.com/gorilla/mux"
"github.com/safing/portbase/log"
@ -21,6 +24,11 @@ var (
// main server and lock
server = &http.Server{}
handlerLock sync.RWMutex
allowedDevCORSOrigins = []string{
"127.0.0.1",
"localhost",
}
)
// RegisterHandler registers a handler with the API endoint.
@ -139,6 +147,12 @@ func (mh *mainHandler) handle(w http.ResponseWriter, r *http.Request) error {
}
// Add security headers.
w.Header().Set("Referrer-Policy", "no-referrer")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("X-Frame-Options", "deny")
w.Header().Set("X-XSS-Protection", "1; mode=block")
w.Header().Set("X-DNS-Prefetch-Control", "off")
// Add CSP Header in production mode.
if !devMode() {
w.Header().Set(
"Content-Security-Policy",
@ -147,13 +161,12 @@ func (mh *mainHandler) handle(w http.ResponseWriter, r *http.Request) error {
"style-src 'self' 'unsafe-inline'; "+
"img-src 'self' data:",
)
w.Header().Set("Referrer-Policy", "no-referrer")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("X-Frame-Options", "deny")
w.Header().Set("X-XSS-Protection", "1; mode=block")
w.Header().Set("X-DNS-Prefetch-Control", "off")
} else {
w.Header().Set("Access-Control-Allow-Origin", "*")
} else if origin := r.Header.Get("Origin"); origin != "" {
// Allow cross origin requests from localhost in dev mode.
if u, err := url.Parse(origin); err == nil &&
utils.StringInSlice(allowedDevCORSOrigins, u.Host) {
w.Header().Set("Access-Control-Allow-Origin", origin)
}
}
// Handle request.

3
config/option.go
View file

@ -9,6 +9,7 @@ import (
"github.com/tidwall/sjson"
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/formats/dsd"
)
// OptionType defines the value type of an option.
@ -303,7 +304,7 @@ func (option *Option) export() (record.Record, error) {
}
}
r, err := record.NewWrapper(fmt.Sprintf("config:%s", option.Key), nil, record.JSON, data)
r, err := record.NewWrapper(fmt.Sprintf("config:%s", option.Key), nil, dsd.JSON, data)
if err != nil {
return nil, err
}

4
database/record/base.go
View file

@ -122,14 +122,14 @@ func (b *Base) MarshalRecord(self Record) ([]byte, error) {
c := container.New([]byte{1})
// meta encoding
metaSection, err := dsd.Dump(b.meta, GenCode)
metaSection, err := dsd.Dump(b.meta, dsd.GenCode)
if err != nil {
return nil, err
}
c.AppendAsBlock(metaSection)
// data
dataSection, err := b.Marshal(self, JSON)
dataSection, err := b.Marshal(self, dsd.JSON)
if err != nil {
return nil, err
}

15
database/record/formats.go
View file

@ -1,15 +0,0 @@
package record
import (
"github.com/safing/portbase/formats/dsd"
)
// Reimport DSD storage types
const (
AUTO = dsd.AUTO
STRING = dsd.STRING // S
BYTES = dsd.BYTES // X
JSON = dsd.JSON // J
BSON = dsd.BSON // B
GenCode = dsd.GenCode // G
)

4
database/record/meta-bench_test.go
View file

@ -432,7 +432,7 @@ func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := dsd.Dump(testMeta, JSON)
_, err := dsd.Dump(testMeta, dsd.JSON)
if err != nil {
b.Errorf("failed to serialize with DSD/JSON: %s", err)
return
@ -444,7 +444,7 @@ func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
// Setup
encodedData, err := dsd.Dump(testMeta, JSON)
encodedData, err := dsd.Dump(testMeta, dsd.JSON)
if err != nil {
b.Errorf("failed to serialize with DSD/JSON: %s", err)
return

12
database/record/wrapper.go
View file

@ -42,7 +42,7 @@ func NewRawWrapper(database, key string, data []byte) (*Wrapper, error) {
return nil, fmt.Errorf("could not unmarshal meta section: %s", err)
}
var format uint8 = dsd.NONE
var format uint8 = dsd.RAW
if !newMeta.IsDeleted() {
format, n, err = varint.Unpack8(data[offset:])
if err != nil {
@ -89,7 +89,7 @@ func (w *Wrapper) Marshal(r Record, format uint8) ([]byte, error) {
return nil, nil
}
if format != AUTO && format != w.Format {
if format != dsd.AUTO && format != w.Format {
return nil, errors.New("could not dump model, wrapped object format mismatch")
}
@ -112,14 +112,14 @@ func (w *Wrapper) MarshalRecord(r Record) ([]byte, error) {
c := container.New([]byte{1})
// meta
metaSection, err := dsd.Dump(w.meta, GenCode)
metaSection, err := dsd.Dump(w.meta, dsd.GenCode)
if err != nil {
return nil, err
}
c.AppendAsBlock(metaSection)
// data
dataSection, err := w.Marshal(r, JSON)
dataSection, err := w.Marshal(r, dsd.JSON)
if err != nil {
return nil, err
}
@ -140,7 +140,7 @@ func Unwrap(wrapped, new Record) error {
return fmt.Errorf("cannot unwrap %T", wrapped)
}
_, err := dsd.LoadAsFormat(wrapper.Data, wrapper.Format, new)
err := dsd.LoadAsFormat(wrapper.Data, wrapper.Format, new)
if err != nil {
return fmt.Errorf("failed to unwrap %T: %s", new, err)
}
@ -153,7 +153,7 @@ func Unwrap(wrapped, new Record) error {
// GetAccessor returns an accessor for this record, if available.
func (w *Wrapper) GetAccessor(self Record) accessor.Accessor {
if w.Format == JSON && len(w.Data) > 0 {
if w.Format == dsd.JSON && len(w.Data) > 0 {
return accessor.NewJSONBytesAccessor(&w.Data)
}
return nil

8
database/record/wrapper_test.go
View file

@ -3,6 +3,8 @@ package record
import (
"bytes"
"testing"
"github.com/safing/portbase/formats/dsd"
)
func TestWrapper(t *testing.T) {
@ -18,18 +20,18 @@ func TestWrapper(t *testing.T) {
encodedTestData := []byte(`J{"a": "b"}`)
// test wrapper
wrapper, err := NewWrapper("test:a", &Meta{}, JSON, testData)
wrapper, err := NewWrapper("test:a", &Meta{}, dsd.JSON, testData)
if err != nil {
t.Fatal(err)
}
if wrapper.Format != JSON {
if wrapper.Format != dsd.JSON {
t.Error("format mismatch")
}
if !bytes.Equal(testData, wrapper.Data) {
t.Error("data mismatch")
}
encoded, err := wrapper.Marshal(wrapper, JSON)
encoded, err := wrapper.Marshal(wrapper, dsd.JSON)
if err != nil {
t.Fatal(err)
}

47
formats/dsd/compression.go
View file

@ -4,26 +4,24 @@ import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"github.com/safing/portbase/formats/varint"
)
// DumpAndCompress stores the interface as a dsd formatted data structure and compresses the resulting data.
func DumpAndCompress(t interface{}, format uint8, compression uint8) ([]byte, error) {
// Check if compression format is valid.
compression, ok := ValidateCompressionFormat(compression)
if !ok {
return nil, ErrIncompatibleFormat
}
// Dump the given data with the given format.
data, err := Dump(t, format)
if err != nil {
return nil, err
}
// handle special cases
switch compression {
case NONE:
return data, nil
case AUTO:
compression = GZIP
}
// prepare writer
packetFormat := varint.Pack8(compression)
buf := bytes.NewBuffer(nil)
@ -53,52 +51,53 @@ func DumpAndCompress(t interface{}, format uint8, compression uint8) ([]byte, er
return nil, err
}
default:
return nil, fmt.Errorf("dsd: tried to compress with unknown format %d", format)
return nil, ErrIncompatibleFormat
}
return buf.Bytes(), nil
}
// DecompressAndLoad decompresses the data using the specified compression format and then loads the resulting data blob into the interface.
func DecompressAndLoad(data []byte, format uint8, t interface{}) (interface{}, error) {
func DecompressAndLoad(data []byte, compression uint8, t interface{}) (format uint8, err error) {
// Check if compression format is valid.
_, ok := ValidateCompressionFormat(compression)
if !ok {
return 0, ErrIncompatibleFormat
}
// prepare reader
buf := bytes.NewBuffer(nil)
// decompress
switch format {
switch compression {
case GZIP:
// create gzip reader
gzipReader, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return nil, err
return 0, err
}
// read uncompressed data
_, err = buf.ReadFrom(gzipReader)
if err != nil {
return nil, err
return 0, err
}
// flush and verify gzip footer
err = gzipReader.Close()
if err != nil {
return nil, err
return 0, err
}
default:
return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
return 0, ErrIncompatibleFormat
}
// assign decompressed data
data = buf.Bytes()
// get format
format, read, err := varint.Unpack8(data)
format, read, err := loadFormat(data)
if err != nil {
return nil, err
return 0, err
}
if len(data) <= read {
return nil, errNoMoreSpace
}
return LoadAsFormat(data[read:], format, t)
return format, LoadAsFormat(data[read:], format, t)
}

144
formats/dsd/dsd.go
View file

@ -7,97 +7,79 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"github.com/fxamacker/cbor/v2"
"github.com/vmihailenco/msgpack/v5"
"github.com/safing/portbase/formats/varint"
"github.com/safing/portbase/utils"
)
// Types.
const (
AUTO = 0
NONE = 1
// Special types.
LIST = 76 // L
// Serialization types.
BSON = 66 // B
CBOR = 67 // C
GenCode = 71 // G
JSON = 74 // J
STRING = 83 // S
BYTES = 88 // X
// Compression types.
GZIP = 90 // Z
)
// Errors.
var (
errNoMoreSpace = errors.New("dsd: no more space left after reading dsd type")
errNotImplemented = errors.New("dsd: this type is not yet implemented")
)
// Load loads an dsd structured data blob into the given interface.
func Load(data []byte, t interface{}) (interface{}, error) {
format, read, err := varint.Unpack8(data)
func Load(data []byte, t interface{}) (format uint8, err error) {
format, read, err := loadFormat(data)
if err != nil {
return nil, err
}
if len(data) <= read {
return nil, errNoMoreSpace
return 0, err
}
switch format {
case GZIP:
return DecompressAndLoad(data[read:], format, t)
default:
return LoadAsFormat(data[read:], format, t)
_, ok := ValidateSerializationFormat(format)
if ok {
return format, LoadAsFormat(data[read:], format, t)
}
return DecompressAndLoad(data[read:], format, t)
}
// LoadAsFormat loads a data blob into the interface using the specified format.
func LoadAsFormat(data []byte, format uint8, t interface{}) (interface{}, error) {
func LoadAsFormat(data []byte, format uint8, t interface{}) (err error) {
switch format {
case STRING:
return string(data), nil
case BYTES:
return data, nil
case RAW:
return ErrIsRaw
case JSON:
err := json.Unmarshal(data, t)
err = json.Unmarshal(data, t)
if err != nil {
return nil, fmt.Errorf("dsd: failed to unpack json: %s, data: %s", err, utils.SafeFirst16Bytes(data))
return fmt.Errorf("dsd: failed to unpack json: %w, data: %s", err, utils.SafeFirst16Bytes(data))
}
return t, nil
case BSON:
return nil, errNotImplemented
// err := bson.Unmarshal(data[read:], t)
// if err != nil {
// return nil, err
// }
// return t, nil
return nil
case CBOR:
err := cbor.Unmarshal(data, t)
err = cbor.Unmarshal(data, t)
if err != nil {
return nil, fmt.Errorf("dsd: failed to unpack cbor: %s, data: %s", err, utils.SafeFirst16Bytes(data))
return fmt.Errorf("dsd: failed to unpack cbor: %w, data: %s", err, utils.SafeFirst16Bytes(data))
}
return t, nil
return nil
case MsgPack:
err = msgpack.Unmarshal(data, t)
if err != nil {
return fmt.Errorf("dsd: failed to unpack msgpack: %w, data: %s", err, utils.SafeFirst16Bytes(data))
}
return nil
case GenCode:
genCodeStruct, ok := t.(GenCodeCompatible)
if !ok {
return nil, errors.New("dsd: gencode is not supported by the given data structure")
return errors.New("dsd: gencode is not supported by the given data structure")
}
_, err := genCodeStruct.GenCodeUnmarshal(data)
_, err = genCodeStruct.GenCodeUnmarshal(data)
if err != nil {
return nil, fmt.Errorf("dsd: failed to unpack gencode: %s, data: %s", err, utils.SafeFirst16Bytes(data))
return fmt.Errorf("dsd: failed to unpack gencode: %w, data: %s", err, utils.SafeFirst16Bytes(data))
}
return t, nil
return nil
default:
return nil, fmt.Errorf("dsd: tried to load unknown type %d, data: %s", format, utils.SafeFirst16Bytes(data))
return ErrIncompatibleFormat
}
}
func loadFormat(data []byte) (format uint8, read int, err error) {
format, read, err = varint.Unpack8(data)
if err != nil {
return 0, 0, err
}
if len(data) <= read {
return 0, 0, io.ErrUnexpectedEOF
}
return format, read, nil
}
// Dump stores the interface as a dsd formatted data structure.
func Dump(t interface{}, format uint8) ([]byte, error) {
return DumpIndent(t, format, "")
@ -105,25 +87,21 @@ func Dump(t interface{}, format uint8) ([]byte, error) {
// DumpIndent stores the interface as a dsd formatted data structure with indentation, if available.
func DumpIndent(t interface{}, format uint8, indent string) ([]byte, error) {
if format == AUTO {
switch t.(type) {
case string:
format = STRING
case []byte:
format = BYTES
default:
format = JSON
}
format, ok := ValidateSerializationFormat(format)
if !ok {
return nil, ErrIncompatibleFormat
}
f := varint.Pack8(format)
var data []byte
var err error
switch format {
case STRING:
data = []byte(t.(string))
case BYTES:
data = t.([]byte)
case RAW:
var ok bool
data, ok = t.([]byte)
if !ok {
return nil, ErrIncompatibleFormat
}
case JSON:
// TODO: use SetEscapeHTML(false)
if indent != "" {
@ -134,17 +112,16 @@ func DumpIndent(t interface{}, format uint8, indent string) ([]byte, error) {
if err != nil {
return nil, err
}
case BSON:
return nil, errNotImplemented
// data, err = bson.Marshal(t)
// if err != nil {
// return nil, err
// }
case CBOR:
data, err = cbor.Marshal(t)
if err != nil {
return nil, err
}
case MsgPack:
data, err = msgpack.Marshal(t)
if err != nil {
return nil, err
}
case GenCode:
genCodeStruct, ok := t.(GenCodeCompatible)
if !ok {
@ -152,12 +129,13 @@ func DumpIndent(t interface{}, format uint8, indent string) ([]byte, error) {
}
data, err = genCodeStruct.GenCodeMarshal(nil)
if err != nil {
return nil, fmt.Errorf("dsd: failed to pack gencode struct: %s", err)
return nil, fmt.Errorf("dsd: failed to pack gencode struct: %w", err)
}
default:
return nil, fmt.Errorf("dsd: tried to dump with unknown format %d", format)
return nil, ErrIncompatibleFormat
}
r := append(f, data...)
return r, nil
// TODO: Find a better way to do this.
f = append(f, data...)
return f, nil
}

219
formats/dsd/dsd_test.go
View file

@ -1,8 +1,7 @@
//nolint:maligned,unparam,gocyclo,gocognit
//nolint:maligned,gocyclo,gocognit
package dsd
import (
"bytes"
"math/big"
"reflect"
"testing"
@ -57,137 +56,116 @@ type GenCodeTestStruct struct {
Bap *[]byte
}
var (
simpleSubject = &SimpleTestStruct{
"a",
0x01,
}
bString = "b"
bBytes byte = 0x02
complexSubject = &ComplexTestStruct{
-1,
-2,
-3,
-4,
-5,
1,
2,
3,
4,
5,
big.NewInt(6),
"a",
&bString,
[]string{"c", "d", "e"},
&[]string{"f", "g", "h"},
0x01,
&bBytes,
[]byte{0x03, 0x04, 0x05},
&[]byte{0x05, 0x06, 0x07},
map[string]string{
"a": "b",
"c": "d",
"e": "f",
},
&map[string]string{
"g": "h",
"i": "j",
"k": "l",
},
}
genCodeSubject = &GenCodeTestStruct{
-2,
-3,
-4,
-5,
2,
3,
4,
5,
"a",
&bString,
[]string{"c", "d", "e"},
&[]string{"f", "g", "h"},
0x01,
&bBytes,
[]byte{0x03, 0x04, 0x05},
&[]byte{0x05, 0x06, 0x07},
}
)
func TestConversion(t *testing.T) {
compressionFormats := []uint8{NONE, GZIP}
t.Parallel()
compressionFormats := []uint8{AUTO, GZIP}
formats := []uint8{JSON, CBOR, MsgPack}
for _, compression := range compressionFormats {
// STRING
d, err := DumpAndCompress("abc", STRING, compression)
if err != nil {
t.Fatalf("Dump error (string): %s", err)
}
s, err := Load(d, nil)
if err != nil {
t.Fatalf("Load error (string): %s", err)
}
ts := s.(string)
if ts != "abc" {
t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", ts, "abc")
}
// BYTES
d, err = DumpAndCompress([]byte("def"), BYTES, compression)
if err != nil {
t.Fatalf("Dump error (string): %s", err)
}
b, err := Load(d, nil)
if err != nil {
t.Fatalf("Load error (string): %s", err)
}
tb := b.([]byte)
if !bytes.Equal(tb, []byte("def")) {
t.Errorf("Load (string): subject and loaded object are not equal (%v != %v)", tb, []byte("def"))
}
// STRUCTS
simpleSubject := SimpleTestStruct{
"a",
0x01,
}
bString := "b"
var bBytes byte = 0x02
complexSubject := ComplexTestStruct{
-1,
-2,
-3,
-4,
-5,
1,
2,
3,
4,
5,
big.NewInt(6),
"a",
&bString,
[]string{"c", "d", "e"},
&[]string{"f", "g", "h"},
0x01,
&bBytes,
[]byte{0x03, 0x04, 0x05},
&[]byte{0x05, 0x06, 0x07},
map[string]string{
"a": "b",
"c": "d",
"e": "f",
},
&map[string]string{
"g": "h",
"i": "j",
"k": "l",
},
}
genCodeSubject := GenCodeTestStruct{
-2,
-3,
-4,
-5,
2,
3,
4,
5,
"a",
&bString,
[]string{"c", "d", "e"},
&[]string{"f", "g", "h"},
0x01,
&bBytes,
[]byte{0x03, 0x04, 0x05},
&[]byte{0x05, 0x06, 0x07},
}
// test all formats (complex)
formats := []uint8{JSON, CBOR}
for _, format := range formats {
// simple
b, err := DumpAndCompress(&simpleSubject, format, compression)
var b []byte
var err error
if compression != AUTO {
b, err = DumpAndCompress(simpleSubject, format, compression)
} else {
b, err = Dump(simpleSubject, format)
}
if err != nil {
t.Fatalf("Dump error (simple struct): %s", err)
}
o, err := Load(b, &SimpleTestStruct{})
si := &SimpleTestStruct{}
_, err = Load(b, si)
if err != nil {
t.Fatalf("Load error (simple struct): %s", err)
}
if !reflect.DeepEqual(&simpleSubject, o) {
if !reflect.DeepEqual(simpleSubject, si) {
t.Errorf("Load (simple struct): subject does not match loaded object")
t.Errorf("Encoded: %v", string(b))
t.Errorf("Compared: %v == %v", &simpleSubject, o)
t.Errorf("Compared: %v == %v", simpleSubject, si)
}
// complex
b, err = DumpAndCompress(&complexSubject, format, compression)
if compression != AUTO {
b, err = DumpAndCompress(complexSubject, format, compression)
} else {
b, err = Dump(complexSubject, format)
}
if err != nil {
t.Fatalf("Dump error (complex struct): %s", err)
}
o, err = Load(b, &ComplexTestStruct{})
co := &ComplexTestStruct{}
_, err = Load(b, co)
if err != nil {
t.Fatalf("Load error (complex struct): %s", err)
}
co := o.(*ComplexTestStruct)
if complexSubject.I != co.I {
t.Errorf("Load (complex struct): struct.I is not equal (%v != %v)", complexSubject.I, co.I)
}
@ -255,39 +233,46 @@ func TestConversion(t *testing.T) {
}
// test all formats
formats = []uint8{JSON, CBOR, GenCode}
simplifiedFormatTesting := []uint8{JSON, CBOR, MsgPack, GenCode}
for _, format := range simplifiedFormatTesting {
for _, format := range formats {
// simple
b, err := DumpAndCompress(&simpleSubject, format, compression)
var b []byte
var err error
if compression != AUTO {
b, err = DumpAndCompress(simpleSubject, format, compression)
} else {
b, err = Dump(simpleSubject, format)
}
if err != nil {
t.Fatalf("Dump error (simple struct): %s", err)
}
o, err := Load(b, &SimpleTestStruct{})
si := &SimpleTestStruct{}
_, err = Load(b, si)
if err != nil {
t.Fatalf("Load error (simple struct): %s", err)
}
if !reflect.DeepEqual(&simpleSubject, o) {
if !reflect.DeepEqual(simpleSubject, si) {
t.Errorf("Load (simple struct): subject does not match loaded object")
t.Errorf("Encoded: %v", string(b))
t.Errorf("Compared: %v == %v", &simpleSubject, o)
t.Errorf("Compared: %v == %v", simpleSubject, si)
}
// complex
b, err = DumpAndCompress(&genCodeSubject, format, compression)
b, err = DumpAndCompress(genCodeSubject, format, compression)
if err != nil {
t.Fatalf("Dump error (complex struct): %s", err)
}
o, err = Load(b, &GenCodeTestStruct{})
co := &GenCodeTestStruct{}
_, err = Load(b, co)
if err != nil {
t.Fatalf("Load error (complex struct): %s", err)
}
co := o.(*GenCodeTestStruct)
if genCodeSubject.I8 != co.I8 {
t.Errorf("Load (complex struct): struct.I8 is not equal (%v != %v)", genCodeSubject.I8, co.I8)
}

68
formats/dsd/format.go Normal file
View file

@ -0,0 +1,68 @@
package dsd
import "errors"
var (
ErrIncompatibleFormat = errors.New("dsd: format is incompatible with operation")
ErrIsRaw = errors.New("dsd: given data is in raw format")
ErrUnknownFormat = errors.New("dsd: format is unknown")
)
// Format types.
const (
AUTO = 0
// Serialization types.
RAW = 1
CBOR = 67 // C
GenCode = 71 // G
JSON = 74 // J
MsgPack = 77 // M
// Compression types.
GZIP = 90 // Z
// Special types.
LIST = 76 // L
)
var (
DefaultSerializationFormat uint8 = JSON
DefaultCompressionFormat uint8 = GZIP
)
// ValidateSerializationFormat validates if the format is for serialization,
// and returns the validated format as well as the result of the validation.
// If called on the AUTO format, it returns the default serialization format.
func ValidateSerializationFormat(format uint8) (validatedFormat uint8, ok bool) {
switch format {
case AUTO:
return DefaultSerializationFormat, true
case RAW:
return format, true
case CBOR:
return format, true
case GenCode:
return format, true
case JSON:
return format, true
case MsgPack:
return format, true
default:
return 0, false
}
}
// ValidateCompressionFormat validates if the format is for compression,
// and returns the validated format as well as the result of the validation.
// If called on the AUTO format, it returns the default compression format.
func ValidateCompressionFormat(format uint8) (validatedFormat uint8, ok bool) {
switch format {
case AUTO:
return DefaultCompressionFormat, true
case GZIP:
return format, true
default:
return 0, false
}
}

2
formats/dsd/gencode_test.go
View file

@ -1,4 +1,4 @@
//nolint:nakedret,unconvert,gocognit
//nolint:nakedret,unconvert,gocognit,wastedassign,gofumpt
package dsd
import (

129
formats/dsd/http.go Normal file
View file

@ -0,0 +1,129 @@
package dsd
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
)
var (
ErrMissingBody = errors.New("dsd: missing http body")
ErrMissingContentType = errors.New("dsd: missing http content type")
)
const (
httpHeaderContentType = "Content-Type"
)
// LoadFromHTTPRequest loads the data from the body into the given interface.
func LoadFromHTTPRequest(r *http.Request, t interface{}) (format uint8, err error) {
return loadFromHTTP(r.Body, r.Header.Get(httpHeaderContentType), t)
}
// LoadFromHTTPResponse loads the data from the body into the given interface.
// Closing the body is left to the caller.
func LoadFromHTTPResponse(resp *http.Response, t interface{}) (format uint8, err error) {
return loadFromHTTP(resp.Body, resp.Header.Get(httpHeaderContentType), t)
}
func loadFromHTTP(body io.Reader, mimeType string, t interface{}) (format uint8, err error) {
// Read full body.
data, err := ioutil.ReadAll(body)
if err != nil {
return 0, fmt.Errorf("dsd: failed to read http body: %w", err)
}
// Get mime type from header, then check, clean and verify it.
if mimeType == "" {
return 0, ErrMissingContentType
}
mimeType, _, err = mime.ParseMediaType(mimeType)
if err != nil {
return 0, fmt.Errorf("dsd: failed to parse content type: %w", err)
}
format, ok := MimeTypeToFormat[mimeType]
if !ok {
return 0, ErrIncompatibleFormat
}
// Parse data..
return format, LoadAsFormat(data, format, t)
}
// RequestHTTPResponseFormat sets the Accept header to the given format.
func RequestHTTPResponseFormat(r *http.Request, format uint8) (mimeType string, err error) {
// Get mime type.
mimeType, ok := FormatToMimeType[format]
if !ok {
return "", ErrIncompatibleFormat
}
// Request response format.
r.Header.Set("Accept", mimeType)
return mimeType, nil
}
// DumpToHTTPRequest dumps the given data to the HTTP request using the given
// format. It also sets the Accept header to the same format.
func DumpToHTTPRequest(r *http.Request, t interface{}, format uint8) error {
mimeType, err := RequestHTTPResponseFormat(r, format)
if err != nil {
return err
}
// Serialize data.
data, err := Dump(t, format)
if err != nil {
return fmt.Errorf("dsd: failed to serialize: %w", err)
}
// Set body.
r.Header.Set("Content-Type", mimeType)
r.Body = ioutil.NopCloser(bytes.NewReader(data))
return nil
}
// DumpToHTTPResponse dumpts the given data to the HTTP response, using the
// format defined in the request's Accept header.
func DumpToHTTPResponse(w http.ResponseWriter, r *http.Request, t interface{}) error {
// Get format from Accept header.
// TODO: Improve parsing of Accept header.
mimeType := r.Header.Get("Accept")
format, ok := MimeTypeToFormat[mimeType]
if !ok {
return ErrIncompatibleFormat
}
// Serialize data.
data, err := Dump(t, format)
if err != nil {
return fmt.Errorf("dsd: failed to serialize: %w", err)
}
// Write data to response
w.Header().Set("Content-Type", mimeType)
_, err = w.Write(data)
if err != nil {
return fmt.Errorf("dsd: failed to write response: %w", err)
}
return nil
}
var (
FormatToMimeType = map[uint8]string{
JSON: "application/json; charset=utf-8",
CBOR: "application/cbor",
MsgPack: "application/msgpack",
}
MimeTypeToFormat = map[string]uint8{
"application/json": JSON,
"application/cbor": CBOR,
"application/msgpack": MsgPack,
}
)

5
go.mod
View file

@ -4,7 +4,7 @@ go 1.15
require (
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/VictoriaMetrics/metrics v1.18.0
github.com/VictoriaMetrics/metrics v1.18.1
github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6
github.com/andyleap/gencode v0.0.0-20171124163308-e1423834d4b4 // indirect
github.com/andyleap/parser v0.0.0-20160126201130-db5a13a7cd46 // indirect
@ -33,8 +33,9 @@ require (
github.com/tidwall/gjson v1.11.0
github.com/tidwall/sjson v1.2.3
github.com/tklauser/go-sysconf v0.3.9 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5
go.etcd.io/bbolt v1.3.6
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e // indirect
golang.org/x/net v0.0.0-20211116231205-47ca1ff31462 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

9
go.sum
View file

@ -25,6 +25,8 @@ github.com/VictoriaMetrics/metrics v1.15.2 h1:w/GD8L9tm+gvx1oZvAofRRXwammiicdI0j
github.com/VictoriaMetrics/metrics v1.15.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
github.com/VictoriaMetrics/metrics v1.18.0 h1:vov5NxDHRSXFbdiH4dYLYEjKLoAXXSQ7hcnG8TSD9JQ=
github.com/VictoriaMetrics/metrics v1.18.0/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0=
github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 h1:5L8Mj9Co9sJVgW3TpYk2gxGJnDjsYuboNTcRmbtGKGs=
github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6/go.mod h1:3HgLJ9d18kXMLQlJvIY3+FszZYMxCz8WfE2MQ7hDY0w=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@ -319,6 +321,11 @@ github.com/valyala/histogram v1.1.2 h1:vOk5VrGjMBIoPR5k6wA8vBaC8toeJ8XO0yfRjFEc1
github.com/valyala/histogram v1.1.2/go.mod h1:CZAr6gK9dbD7hYx2s8WSPh0p5x5wETjC+2b3PJVtEdg=
github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@ -375,6 +382,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxW
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e h1:Xj+JO91noE97IN6F/7WZxzC5QE6yENAQPrwIYhW3bsA=
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211116231205-47ca1ff31462 h1:2vmJlzGKvQ7e/X9XT0XydeWDxmqx8DnegiIMRT+5ssI=
golang.org/x/net v0.0.0-20211116231205-47ca1ff31462/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=