Add support for mime types and checksums to import/export

sync/module.go

@@ -1,10 +1,6 @@
 package sync
 
 import (
-	"errors"
-	"net/http"
-
-	"github.com/safing/portbase/api"
 	"github.com/safing/portbase/database"
 	"github.com/safing/portbase/modules"
 )
@@ -31,84 +27,3 @@ func prep() error {
 	}
 	return nil
 }
-
-// Type is the type of an export.
-type Type string
-
-// Export Types.
-const (
-	TypeProfile       = "profile"
-	TypeSettings      = "settings"
-	TypeSingleSetting = "single-setting"
-)
-
-// Export IDs.
-const (
-	ExportTargetGlobal = "global"
-)
-
-// Messages.
-var (
-	MsgNone           = ""
-	MsgValid          = "Import is valid."
-	MsgSuccess        = "Import successful."
-	MsgRequireRestart = "Import successful. Restart required for setting to take effect."
-)
-
-// ExportRequest is a request for an export.
-type ExportRequest struct {
-	From string `json:"from"`
-	Key  string `json:"key"`
-}
-
-// ImportRequest is a request to import an export.
-type ImportRequest struct {
-	// Where the export should be import to.
-	Target string `json:"target"`
-	// Only validate, but do not actually change anything.
-	ValidateOnly bool `json:"validate_only"`
-
-	RawExport string `json:"raw_export"`
-}
-
-// ImportResult is returned by successful import operations.
-type ImportResult struct {
-	RestartRequired  bool `json:"restart_required"`
-	ReplacesExisting bool `json:"replaces_existing"`
-}
-
-// Errors.
-var (
-	ErrMismatch = api.ErrorWithStatus(
-		errors.New("the supplied export cannot be imported here"),
-		http.StatusPreconditionFailed,
-	)
-	ErrTargetNotFound = api.ErrorWithStatus(
-		errors.New("import/export target does not exist"),
-		http.StatusGone,
-	)
-	ErrUnchanged = api.ErrorWithStatus(
-		errors.New("cannot export unchanged setting"),
-		http.StatusGone,
-	)
-	ErrInvalidImport = api.ErrorWithStatus(
-		errors.New("invalid import"),
-		http.StatusUnprocessableEntity,
-	)
-	ErrInvalidSetting = api.ErrorWithStatus(
-		errors.New("invalid setting"),
-		http.StatusUnprocessableEntity,
-	)
-	ErrInvalidProfile = api.ErrorWithStatus(
-		errors.New("invalid profile"),
-		http.StatusUnprocessableEntity,
-	)
-	ErrImportFailed = api.ErrorWithStatus(
-		errors.New("import failed"),
-		http.StatusInternalServerError,
-	)
-	ErrExportFailed = api.ErrorWithStatus(
-		errors.New("export failed"),
-		http.StatusInternalServerError,
-	)
-)

sync/setting_single.go

@@ -6,10 +6,9 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/ghodss/yaml"
-
 	"github.com/safing/portbase/api"
 	"github.com/safing/portbase/config"
+	"github.com/safing/portbase/formats/dsd"
 	"github.com/safing/portmaster/profile"
 )
 
@@ -91,7 +90,7 @@ func handleExportSingleSetting(ar *api.Request) (data []byte, err error) {
 	} else {
 		request = &ExportRequest{}
 		if err := json.Unmarshal(ar.InputData, request); err != nil {
-			return nil, fmt.Errorf("%w: failed to parse export request: %s", ErrExportFailed, err)
+			return nil, fmt.Errorf("%w: failed to parse export request: %w", ErrExportFailed, err)
 		}
 	}
 
@@ -106,15 +105,7 @@ func handleExportSingleSetting(ar *api.Request) (data []byte, err error) {
 		return nil, err
 	}
 
-	// Make some yummy yaml.
-	yamlData, err := yaml.Marshal(export)
-	if err != nil {
-		return nil, fmt.Errorf("%w: failed to marshal to yaml: %s", ErrExportFailed, err)
-	}
-
-	// TODO: Add checksum for integrity.
-
-	return yamlData, nil
+	return serializeExport(export, ar)
 }
 
 func handleImportSingleSetting(ar *api.Request) (any, error) {
@@ -128,31 +119,35 @@ func handleImportSingleSetting(ar *api.Request) (any, error) {
 				Target:       q.Get("to"),
 				ValidateOnly: q.Has("validate"),
 				RawExport:    string(ar.InputData),
+				RawMime:      ar.Header.Get("Content-Type"),
 			},
 		}
 	} else {
 		request = &SingleSettingImportRequest{}
-		if err := json.Unmarshal(ar.InputData, request); err != nil {
-			return nil, fmt.Errorf("%w: failed to parse import request: %s", ErrInvalidImport, err)
+		if _, err := dsd.MimeLoad(ar.InputData, ar.Header.Get("Accept"), request); err != nil {
+			return nil, fmt.Errorf("%w: failed to parse import request: %w", ErrInvalidImportRequest, err)
 		}
 	}
 
 	// Check if we need to parse the export.
 	switch {
 	case request.Export != nil && request.RawExport != "":
-		return nil, fmt.Errorf("%w: both Export and RawExport are defined", ErrInvalidImport)
+		return nil, fmt.Errorf("%w: both Export and RawExport are defined", ErrInvalidImportRequest)
 	case request.RawExport != "":
-		// TODO: Verify checksum for integrity.
-
+		// Parse export.
 		export := &SingleSettingExport{}
-		if err := yaml.Unmarshal([]byte(request.RawExport), export); err != nil {
-			return nil, fmt.Errorf("%w: failed to parse export: %s", ErrInvalidImport, err)
+		if err := parseExport(&request.ImportRequest, export); err != nil {
+			return nil, err
 		}
 		request.Export = export
+	case request.Export != nil:
+		// Export is aleady parsed.
+	default:
+		return nil, ErrInvalidImportRequest
 	}
 
 	// Optional check if the setting key matches.
-	if q.Has("key") && q.Get("key") != request.Export.ID {
+	if len(q) > 0 && q.Has("key") && q.Get("key") != request.Export.ID {
 		return nil, ErrMismatch
 	}
 
@@ -162,25 +157,32 @@ func handleImportSingleSetting(ar *api.Request) (any, error) {
 
 // ExportSingleSetting export a single setting.
 func ExportSingleSetting(key, from string) (*SingleSettingExport, error) {
+	option, err := config.GetOption(key)
+	if err != nil {
+		return nil, fmt.Errorf("%w: configuration %w", ErrSettingNotFound, err)
+	}
+
 	var value any
 	if from == ExportTargetGlobal {
-		option, err := config.GetOption(key)
-		if err != nil {
-			return nil, fmt.Errorf("%w: configuration %s", ErrTargetNotFound, err)
-		}
 		value = option.UserValue()
 		if value == nil {
 			return nil, ErrUnchanged
 		}
 	} else {
+		// Check if the setting is settable per app.
+		if !option.AnnotationEquals(config.SettablePerAppAnnotation, true) {
+			return nil, ErrNotSettablePerApp
+		}
+		// Get and load profile.
 		r, err := db.Get(profile.ProfilesDBPath + from)
 		if err != nil {
-			return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
+			return nil, fmt.Errorf("%w: failed to find profile: %w", ErrTargetNotFound, err)
 		}
 		p, err := profile.EnsureProfile(r)
 		if err != nil {
-			return nil, fmt.Errorf("%w: failed to load profile: %s", ErrExportFailed, err)
+			return nil, fmt.Errorf("%w: failed to load profile: %w", ErrExportFailed, err)
 		}
+		// Flatten config and get key we are looking for.
 		flattened := config.Flatten(p.Config)
 		value = flattened[key]
 		if value == nil {
@@ -205,10 +207,10 @@ func ImportSingeSetting(r *SingleSettingImportRequest) (*ImportResult, error) {
 	// Get option and validate value.
 	option, err := config.GetOption(r.Export.ID)
 	if err != nil {
-		return nil, fmt.Errorf("%w: configuration %s", ErrTargetNotFound, err)
+		return nil, fmt.Errorf("%w: configuration %w", ErrSettingNotFound, err)
 	}
-	if option.ValidateValue(r.Export.Value) != nil {
-		return nil, fmt.Errorf("%w: configuration value is invalid: %s", ErrInvalidSetting, err)
+	if err := option.ValidateValue(r.Export.Value); err != nil {
+		return nil, fmt.Errorf("%w: %w", ErrInvalidSettingValue, err)
 	}
 
 	// Import single global setting.
@@ -222,19 +224,22 @@ func ImportSingeSetting(r *SingleSettingImportRequest) (*ImportResult, error) {
 		}
 
 		// Actually import the setting.
-		err = config.SetConfigOption(r.Export.ID, r.Export.Value)
-		if err != nil {
-			return nil, fmt.Errorf("%w: configuration value is invalid: %s", ErrInvalidSetting, err)
+		if err := config.SetConfigOption(r.Export.ID, r.Export.Value); err != nil {
+			return nil, fmt.Errorf("%w: %w", ErrInvalidSettingValue, err)
 		}
 	} else {
+		// Check if the setting is settable per app.
+		if !option.AnnotationEquals(config.SettablePerAppAnnotation, true) {
+			return nil, ErrNotSettablePerApp
+		}
 		// Import single setting into profile.
 		rec, err := db.Get(profile.ProfilesDBPath + r.Target)
 		if err != nil {
-			return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
+			return nil, fmt.Errorf("%w: failed to find profile: %w", ErrTargetNotFound, err)
 		}
 		p, err := profile.EnsureProfile(rec)
 		if err != nil {
-			return nil, fmt.Errorf("%w: failed to load profile: %s", ErrImportFailed, err)
+			return nil, fmt.Errorf("%w: failed to load profile: %w", ErrImportFailed, err)
 		}
 
 		// Stop here if we are only validating.
@@ -246,14 +251,11 @@ func ImportSingeSetting(r *SingleSettingImportRequest) (*ImportResult, error) {
 		}
 
 		// Set imported setting on profile.
-		flattened := config.Flatten(p.Config)
-		flattened[r.Export.ID] = r.Export.Value
-		p.Config = config.Expand(flattened)
+		config.PutValueIntoHierarchicalConfig(p.Config, r.Export.ID, r.Export.Value)
 
 		// Save profile back to db.
-		err = p.Save()
-		if err != nil {
-			return nil, fmt.Errorf("%w: failed to save profile: %s", ErrImportFailed, err)
+		if err := p.Save(); err != nil {
+			return nil, fmt.Errorf("%w: failed to save profile: %w", ErrImportFailed, err)
 		}
 	}
 

sync/settings.go

@@ -7,8 +7,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/ghodss/yaml"
-
 	"github.com/safing/portbase/api"
 	"github.com/safing/portbase/config"
 	"github.com/safing/portmaster/profile"
@@ -91,7 +89,7 @@ func handleExportSettings(ar *api.Request) (data []byte, err error) {
 	} else {
 		request = &ExportRequest{}
 		if err := json.Unmarshal(ar.InputData, request); err != nil {
-			return nil, fmt.Errorf("%w: failed to parse export request: %s", ErrExportFailed, err)
+			return nil, fmt.Errorf("%w: failed to parse export request: %w", ErrExportFailed, err)
 		}
 	}
 
@@ -106,15 +104,7 @@ func handleExportSettings(ar *api.Request) (data []byte, err error) {
 		return nil, err
 	}
 
-	// Make some yummy yaml.
-	yamlData, err := yaml.Marshal(export)
-	if err != nil {
-		return nil, fmt.Errorf("%w: failed to marshal to yaml: %s", ErrExportFailed, err)
-	}
-
-	// TODO: Add checksum for integrity.
-
-	return yamlData, nil
+	return serializeExport(export, ar)
 }
 
 func handleImportSettings(ar *api.Request) (any, error) {
@@ -128,28 +118,32 @@ func handleImportSettings(ar *api.Request) (any, error) {
 				Target:       q.Get("to"),
 				ValidateOnly: q.Has("validate"),
 				RawExport:    string(ar.InputData),
+				RawMime:      ar.Header.Get("Content-Type"),
 			},
 			Reset: q.Has("reset"),
 		}
 	} else {
 		request = &SettingsImportRequest{}
 		if err := json.Unmarshal(ar.InputData, request); err != nil {
-			return nil, fmt.Errorf("%w: failed to parse import request: %s", ErrInvalidImport, err)
+			return nil, fmt.Errorf("%w: failed to parse import request: %w", ErrInvalidImportRequest, err)
 		}
 	}
 
 	// Check if we need to parse the export.
 	switch {
 	case request.Export != nil && request.RawExport != "":
-		return nil, fmt.Errorf("%w: both Export and RawExport are defined", ErrInvalidImport)
+		return nil, fmt.Errorf("%w: both Export and RawExport are defined", ErrInvalidImportRequest)
 	case request.RawExport != "":
-		// TODO: Verify checksum for integrity.
-
+		// Parse export.
 		export := &SettingsExport{}
-		if err := yaml.Unmarshal([]byte(request.RawExport), export); err != nil {
-			return nil, fmt.Errorf("%w: failed to parse export: %s", ErrInvalidImport, err)
+		if err := parseExport(&request.ImportRequest, export); err != nil {
+			return nil, err
 		}
 		request.Export = export
+	case request.Export != nil:
+		// Export is aleady parsed.
+	default:
+		return nil, ErrInvalidImportRequest
 	}
 
 	// Import.
@@ -172,11 +166,11 @@ func ExportSettings(from string) (*SettingsExport, error) {
 	} else {
 		r, err := db.Get(profile.ProfilesDBPath + from)
 		if err != nil {
-			return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
+			return nil, fmt.Errorf("%w: failed to find profile: %w", ErrTargetNotFound, err)
 		}
 		p, err := profile.EnsureProfile(r)
 		if err != nil {
-			return nil, fmt.Errorf("%w: failed to load profile: %s", ErrExportFailed, err)
+			return nil, fmt.Errorf("%w: failed to load profile: %w", ErrExportFailed, err)
 		}
 		settings = config.Flatten(p.Config)
 	}
@@ -207,8 +201,9 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) {
 
 	// Validate config and gather some metadata.
 	var (
-		result  = &ImportResult{}
-		checked int
+		result                 = &ImportResult{}
+		checked                int
+		globalOnlySettingFound bool
 	)
 	err := config.ForEachOption(func(option *config.Option) error {
 		// Check if any setting is set.
@@ -222,7 +217,7 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) {
 
 			// Validate the new value.
 			if err := option.ValidateValue(newValue); err != nil {
-				return fmt.Errorf("%w: configuration value for %s is invalid: %s", ErrInvalidSetting, option.Key, err)
+				return fmt.Errorf("%w: configuration value for %s is invalid: %w", ErrInvalidSettingValue, option.Key, err)
 			}
 
 			// Collect metadata.
@@ -232,6 +227,9 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) {
 			if !r.Reset && option.IsSetByUser() {
 				result.ReplacesExisting = true
 			}
+			if !option.AnnotationEquals(config.SettablePerAppAnnotation, true) {
+				globalOnlySettingFound = true
+			}
 		}
 		return nil
 	})
@@ -239,7 +237,7 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) {
 		return nil, err
 	}
 	if checked < len(settings) {
-		return nil, fmt.Errorf("%w: the export contains unknown settings", ErrInvalidImport)
+		return nil, fmt.Errorf("%w: the export contains unknown settings", ErrInvalidImportRequest)
 	}
 
 	// Import global settings.
@@ -267,18 +265,21 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) {
 		return result, nil
 	}
 
-	// Import settings into profile.
+	// Check if a setting is settable per app.
+	if globalOnlySettingFound {
+		return nil, fmt.Errorf("%w: export contains settings that cannot be set per app", ErrNotSettablePerApp)
+	}
+
+	// Get and load profile.
 	rec, err := db.Get(profile.ProfilesDBPath + r.Target)
 	if err != nil {
-		return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
+		return nil, fmt.Errorf("%w: failed to find profile: %w", ErrTargetNotFound, err)
 	}
 	p, err := profile.EnsureProfile(rec)
 	if err != nil {
-		return nil, fmt.Errorf("%w: failed to load profile: %s", ErrImportFailed, err)
+		return nil, fmt.Errorf("%w: failed to load profile: %w", ErrImportFailed, err)
 	}
 
-	// FIXME: check if there are any global-only setting in the import
-
 	// Stop here if we are only validating.
 	if r.ValidateOnly {
 		return result, nil
@@ -288,17 +289,15 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) {
 	if r.Reset {
 		p.Config = config.Expand(settings)
 	} else {
-		flattenedProfileConfig := config.Flatten(p.Config)
 		for k, v := range settings {
-			flattenedProfileConfig[k] = v
+			config.PutValueIntoHierarchicalConfig(p.Config, k, v)
 		}
-		p.Config = config.Expand(flattenedProfileConfig)
 	}
 
 	// Save profile back to db.
 	err = p.Save()
 	if err != nil {
-		return nil, fmt.Errorf("%w: failed to save profile: %s", ErrImportFailed, err)
+		return nil, fmt.Errorf("%w: failed to save profile: %w", ErrImportFailed, err)
 	}
 
 	return result, nil

sync/util.go

@@ -0,0 +1,147 @@
+package sync
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/safing/jess/filesig"
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/formats/dsd"
+)
+
+// Type is the type of an export.
+type Type string
+
+// Export Types.
+const (
+	TypeProfile       = "profile"
+	TypeSettings      = "settings"
+	TypeSingleSetting = "single-setting"
+)
+
+// Export IDs.
+const (
+	ExportTargetGlobal = "global"
+)
+
+// Messages.
+var (
+	MsgNone           = ""
+	MsgValid          = "Import is valid."
+	MsgSuccess        = "Import successful."
+	MsgRequireRestart = "Import successful. Restart required for setting to take effect."
+)
+
+// ExportRequest is a request for an export.
+type ExportRequest struct {
+	From string `json:"from"`
+	Key  string `json:"key"`
+}
+
+// ImportRequest is a request to import an export.
+type ImportRequest struct {
+	// Where the export should be import to.
+	Target string `json:"target"`
+	// Only validate, but do not actually change anything.
+	ValidateOnly bool `json:"validate_only"`
+
+	RawExport string `json:"raw_export"`
+	RawMime   string `json:"raw_mime"`
+}
+
+// ImportResult is returned by successful import operations.
+type ImportResult struct {
+	RestartRequired  bool `json:"restart_required"`
+	ReplacesExisting bool `json:"replaces_existing"`
+}
+
+// Errors.
+var (
+	ErrMismatch = api.ErrorWithStatus(
+		errors.New("the supplied export cannot be imported here"),
+		http.StatusPreconditionFailed,
+	)
+	ErrSettingNotFound = api.ErrorWithStatus(
+		errors.New("setting not found"),
+		http.StatusPreconditionFailed,
+	)
+	ErrTargetNotFound = api.ErrorWithStatus(
+		errors.New("import/export target does not exist"),
+		http.StatusGone,
+	)
+	ErrUnchanged = api.ErrorWithStatus(
+		errors.New("cannot export unchanged setting"),
+		http.StatusGone,
+	)
+	ErrNotSettablePerApp = api.ErrorWithStatus(
+		errors.New("cannot be set per app"),
+		http.StatusGone,
+	)
+	ErrInvalidImportRequest = api.ErrorWithStatus(
+		errors.New("invalid import request"),
+		http.StatusUnprocessableEntity,
+	)
+	ErrInvalidSettingValue = api.ErrorWithStatus(
+		errors.New("invalid setting value"),
+		http.StatusUnprocessableEntity,
+	)
+	ErrInvalidProfileData = api.ErrorWithStatus(
+		errors.New("invalid profile data"),
+		http.StatusUnprocessableEntity,
+	)
+	ErrImportFailed = api.ErrorWithStatus(
+		errors.New("import failed"),
+		http.StatusInternalServerError,
+	)
+	ErrExportFailed = api.ErrorWithStatus(
+		errors.New("export failed"),
+		http.StatusInternalServerError,
+	)
+)
+
+func serializeExport(export any, ar *api.Request) ([]byte, error) {
+	// Serialize data.
+	data, mimeType, format, err := dsd.MimeDump(export, ar.Header.Get("Accept"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to serialize data: %w", err)
+	}
+	ar.ResponseHeader.Set("Content-Type", mimeType)
+
+	// Add checksum.
+	switch format {
+	case dsd.JSON:
+		data, err = filesig.AddJSONChecksum(data)
+	case dsd.YAML:
+		data, err = filesig.AddYAMLChecksum(data, filesig.TextPlacementTop)
+	default:
+		return nil, dsd.ErrIncompatibleFormat
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to add checksum: %w", err)
+	}
+
+	return data, nil
+}
+
+func parseExport(request *ImportRequest, export any) error {
+	format, err := dsd.MimeLoad([]byte(request.RawExport), request.RawMime, export)
+	if err != nil {
+		return fmt.Errorf("%w: failed to parse export: %w", ErrInvalidImportRequest, err)
+	}
+
+	// Verify checksum, if available.
+	switch format {
+	case dsd.JSON:
+		err = filesig.VerifyJSONChecksum([]byte(request.RawExport))
+	case dsd.YAML:
+		err = filesig.VerifyYAMLChecksum([]byte(request.RawExport))
+	default:
+		// Checksums not supported.
+	}
+	if err != nil && errors.Is(err, filesig.ErrChecksumMissing) {
+		return fmt.Errorf("failed to verify checksum: %w", err)
+	}
+
+	return nil
+}