Add support for import/export of profile icon

go.mod

@@ -86,6 +86,7 @@ require (
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/valyala/fastrand v1.1.0 // indirect
 	github.com/valyala/histogram v1.2.0 // indirect
+	github.com/vincent-petithory/dataurl v1.0.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect

go.sum

@@ -260,6 +260,8 @@ github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
 github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
+github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=
+github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U=
 github.com/vmihailenco/msgpack/v5 v5.4.0 h1:hRM0digJwyR6vll33NNAwCFguy5JuBD6jxDmQP3l608=
 github.com/vmihailenco/msgpack/v5 v5.4.0/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=

profile/icon.go

@@ -1,9 +1,16 @@
 package profile
 
 import (
+	"errors"
+	"fmt"
 	"strings"
+	"sync"
 
+	"github.com/vincent-petithory/dataurl"
 	"golang.org/x/exp/slices"
+
+	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/database/record"
 )
 
 // Icon describes an icon.
@@ -58,3 +65,68 @@ func sortAndCompactIcons(icons []Icon) []Icon {
 
 	return icons
 }
+
+// GetIconAsDataURL returns the icon data as a data URL.
+func (icon *Icon) GetIconAsDataURL() (bloburl string, err error) {
+	switch icon.Type {
+	case IconTypeFile:
+		return "", errors.New("getting icon from file is not supported")
+
+	case IconTypeDatabase:
+		if !strings.HasPrefix(icon.Value, "cache:icons/") {
+			return "", errors.New("invalid icon db key")
+		}
+		r, err := iconDB.Get(icon.Value)
+		if err != nil {
+			return "", err
+		}
+		dbIcon, err := EnsureIconInDatabase(r)
+		if err != nil {
+			return "", err
+		}
+		return dbIcon.IconData, nil
+
+	case IconTypeAPI:
+		data, err := GetProfileIcon(icon.Value)
+		if err != nil {
+			return "", err
+		}
+		return dataurl.EncodeBytes(data), nil
+
+	default:
+		return "", errors.New("unknown icon type")
+	}
+}
+
+var iconDB = database.NewInterface(&database.Options{
+	Local:    true,
+	Internal: true,
+})
+
+type IconInDatabase struct {
+	sync.Mutex
+	record.Base
+
+	IconData string `json:"iconData,omitempty"` // DataURL
+}
+
+// EnsureIconInDatabase ensures that the given record is a *IconInDatabase, and returns it.
+func EnsureIconInDatabase(r record.Record) (*IconInDatabase, error) {
+	// unwrap
+	if r.IsWrapped() {
+		// only allocate a new struct, if we need it
+		newIcon := &IconInDatabase{}
+		err := record.Unwrap(r, newIcon)
+		if err != nil {
+			return nil, err
+		}
+		return newIcon, nil
+	}
+
+	// or adjust type
+	newIcon, ok := r.(*IconInDatabase)
+	if !ok {
+		return nil, fmt.Errorf("record not of type *IconInDatabase, but %T", r)
+	}
+	return newIcon, nil
+}

profile/icons.go

@@ -17,6 +17,11 @@ var profileIconStoragePath = ""
 
 // GetProfileIcon returns the profile icon with the given ID and extension.
 func GetProfileIcon(name string) (data []byte, err error) {
+	// Check if enabled.
+	if profileIconStoragePath == "" {
+		return nil, errors.New("api icon storage not configured")
+	}
+
 	// Build storage path.
 	iconPath := filepath.Clean(
 		filepath.Join(profileIconStoragePath, name),

sync/profile.go

@@ -12,49 +12,49 @@ import (
 	"github.com/safing/portbase/config"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portmaster/profile"
+	"github.com/vincent-petithory/dataurl"
 )
 
 // ProfileExport holds an export of a profile.
 type ProfileExport struct { //nolint:maligned
-	Type Type `json:"type"`
+	Type Type `json:"type" yaml:"type"`
 
 	// Identification
-	ID     string                `json:"id,omitempty"`
+	ID     string                `json:"id,omitempty"     yaml:"id,omitempty"`
-	Source profile.ProfileSource `json:"source,omitempty"`
+	Source profile.ProfileSource `json:"source,omitempty" yaml:"source,omitempty"`
 
 	// Human Metadata
-	Name                string        `json:"name"`
+	Name                string `json:"name"                  yaml:"name"`
-	Description         string        `json:"description,omitempty"`
+	Description         string `json:"description,omitempty" yaml:"description,omitempty"`
-	Homepage            string        `json:"homepage,omitempty"`
+	Homepage            string `json:"homepage,omitempty"    yaml:"homepage,omitempty"`
-	Icons               []ProfileIcon `json:"icons,omitempty"`
+	PresentationPath    string `json:"presPath,omitempty"    yaml:"presPath,omitempty"`
-	PresentationPath    string        `json:"presPath,omitempty"`
+	UsePresentationPath bool   `json:"usePresPath,omitempty" yaml:"usePresPath,omitempty"`
-	UsePresentationPath bool          `json:"usePresPath,omitempty"`
+	IconData            string `json:"iconData,omitempty"    yaml:"iconData,omitempty"` // DataURL
 
 	// Process matching
-	Fingerprints []ProfileFingerprint `json:"fingerprints"`
+	Fingerprints []ProfileFingerprint `json:"fingerprints" yaml:"fingerprints"`
 
 	// Settings
-	Config map[string]any `json:"config,omitempty"`
+	Config map[string]any `json:"config,omitempty" yaml:"config,omitempty"`
 
 	// Metadata
-	LastEdited *time.Time `json:"lastEdited,omitempty"`
+	LastEdited *time.Time `json:"lastEdited,omitempty" yaml:"lastEdited,omitempty"`
-	Created    *time.Time `json:"created,omitempty"`
+	Created    *time.Time `json:"created,omitempty"    yaml:"created,omitempty"`
-	Internal   bool       `json:"internal,omitempty"`
+	Internal   bool       `json:"internal,omitempty"   yaml:"internal,omitempty"`
 }
 
-// ProfileIcon represents a profile icon.
+// ProfileIcon represents a profile icon only.
 type ProfileIcon struct {
-	Type  profile.IconType `json:"type"`
+	IconData string `json:"iconData,omitempty" yaml:"iconData,omitempty"` // DataURL
-	Value string           `json:"value"`
 }
 
 // ProfileFingerprint represents a profile fingerprint.
 type ProfileFingerprint struct {
-	Type       string `json:"type"`
+	Type       string `json:"type"                 yaml:"type"`
-	Key        string `json:"key,omitempty"`
+	Key        string `json:"key,omitempty"        yaml:"key,omitempty"`
-	Operation  string `json:"operation"`
+	Operation  string `json:"operation"            yaml:"operation"`
-	Value      string `json:"value"`
+	Value      string `json:"value"                yaml:"value"`
-	MergedFrom string `json:"mergedFrom,omitempty"`
+	MergedFrom string `json:"mergedFrom,omitempty" yaml:"mergedFrom,omitempty"`
 }
 
 // ProfileExportRequest is a request for a profile export.
@@ -124,7 +124,8 @@ func registerProfileAPI() error {
 				Method:      http.MethodPost,
 				Field:       "allowUnknown",
 				Description: "Allow importing of unknown values.",
-			}},
+			},
+		},
 		BelongsTo:  module,
 		StructFunc: handleImportProfile,
 	}); err != nil {
@@ -161,7 +162,7 @@ func handleExportProfile(ar *api.Request) (data []byte, err error) {
 		return nil, err
 	}
 
-	return serializeExport(export, ar)
+	return serializeProfileExport(export, ar)
 }
 
 func handleImportProfile(ar *api.Request) (any, error) {
@@ -230,7 +231,6 @@ func ExportProfile(scopedID string) (*ProfileExport, error) {
 		Name:                p.Name,
 		Description:         p.Description,
 		Homepage:            p.Homepage,
-		Icons:               convertIconsToExport(p.Icons),
 		PresentationPath:    p.PresentationPath,
 		UsePresentationPath: p.UsePresentationPath,
 
@@ -253,6 +253,22 @@ func ExportProfile(scopedID string) (*ProfileExport, error) {
 		export.Created = &created
 	}
 
+	// Add first exportable icon to export.
+	if len(p.Icons) > 0 {
+		var err error
+		for _, icon := range p.Icons {
+			var iconDataURL string
+			iconDataURL, err = icon.GetIconAsDataURL()
+			if err == nil {
+				export.IconData = iconDataURL
+				break
+			}
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%w: failed to export icon: %w", ErrExportFailed, err)
+		}
+	}
+
 	return export, nil
 }
 
@@ -272,9 +288,8 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil
 	profileID := profile.DeriveProfileID(fingerprints)
 	if r.Export.ID != "" && r.Export.ID != profileID {
 		return nil, ErrMismatch
-	} else {
-		r.Export.ID = profileID
 	}
+	r.Export.ID = profileID
 	// Check Fingerprints.
 	_, err := profile.ParseFingerprints(fingerprints, "")
 	if err != nil {
@@ -349,7 +364,6 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil
 		Name:                in.Name,
 		Description:         in.Description,
 		Homepage:            in.Homepage,
-		Icons:               convertIconsToInternal(in.Icons),
 		PresentationPath:    in.PresentationPath,
 		UsePresentationPath: in.UsePresentationPath,
 
@@ -370,6 +384,22 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil
 		p.Created = in.Created.Unix()
 	}
 
+	// Add icon to profile, if set.
+	if in.IconData != "" {
+		du, err := dataurl.DecodeString(in.IconData)
+		if err != nil {
+			return nil, fmt.Errorf("%w: icon data is invalid: %w", ErrImportFailed, err)
+		}
+		filename, err := profile.UpdateProfileIcon(du.Data, du.MediaType.Subtype)
+		if err != nil {
+			return nil, fmt.Errorf("%w: icon is invalid: %w", ErrImportFailed, err)
+		}
+		p.Icons = []profile.Icon{{
+			Type:  profile.IconTypeAPI,
+			Value: filename,
+		}}
+	}
+
 	// Save profile to db.
 	p.SetKey(profile.MakeProfileKey(p.Source, p.ID))
 	err = p.Save()
@@ -388,28 +418,6 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil
 	return result, nil
 }
 
-func convertIconsToExport(icons []profile.Icon) []ProfileIcon {
-	converted := make([]ProfileIcon, 0, len(icons))
-	for _, icon := range icons {
-		converted = append(converted, ProfileIcon{
-			Type:  icon.Type,
-			Value: icon.Value,
-		})
-	}
-	return converted
-}
-
-func convertIconsToInternal(icons []ProfileIcon) []profile.Icon {
-	converted := make([]profile.Icon, 0, len(icons))
-	for _, icon := range icons {
-		converted = append(converted, profile.Icon{
-			Type:  icon.Type,
-			Value: icon.Value,
-		})
-	}
-	return converted
-}
-
 func convertFingerprintsToExport(fingerprints []profile.Fingerprint) []ProfileFingerprint {
 	converted := make([]ProfileFingerprint, 0, len(fingerprints))
 	for _, fp := range fingerprints {

sync/setting_single.go

@@ -15,10 +15,10 @@ import (
 
 // SingleSettingExport holds an export of a single setting.
 type SingleSettingExport struct {
-	Type Type   `json:"type"` // Must be TypeSingleSetting
+	Type Type   `json:"type" yaml:"type"` // Must be TypeSingleSetting
-	ID   string `json:"id"`   // Settings Key
+	ID   string `json:"id"   yaml:"id"`   // Settings Key
 
-	Value any `json:"value"`
+	Value any `json:"value" yaml:"value"`
 }
 
 // SingleSettingImportRequest is a request to import a single setting.

sync/settings.go

@@ -15,25 +15,25 @@ import (
 
 // SettingsExport holds an export of settings.
 type SettingsExport struct {
-	Type Type `json:"type"`
+	Type Type `json:"type" yaml:"type"`
 
-	Config map[string]any `json:"config"`
+	Config map[string]any `json:"config" yaml:"config"`
 }
 
 // SettingsImportRequest is a request to import settings.
 type SettingsImportRequest struct {
-	ImportRequest `json:",inline"`
+	ImportRequest `json:",inline" yaml:",inline"`
 
 	// Reset all settings of target before import.
 	// The ImportResult also reacts to this flag and correctly reports whether
 	// any settings would be replaced or deleted.
-	Reset bool `json:"reset"`
+	Reset bool `json:"reset" yaml:"reset"`
 
 	// AllowUnknown allows the import of unknown settings.
 	// Otherwise, attempting to import an unknown setting will result in an error.
-	AllowUnknown bool `json:"allowUnknown"`
+	AllowUnknown bool `json:"allowUnknown" yaml:"allowUnknown"`
 
-	Export *SettingsExport `json:"export"`
+	Export *SettingsExport `json:"export" yaml:"export"`
 }
 
 func registerSettingsAPI() error {

sync/util.go

@@ -1,12 +1,16 @@
 package sync
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 
+	yaml "gopkg.in/yaml.v3"
+
 	"github.com/safing/jess/filesig"
 	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/container"
 	"github.com/safing/portbase/formats/dsd"
 )
 
@@ -101,28 +105,85 @@ var (
 	)
 )
 
-func serializeExport(export any, ar *api.Request) ([]byte, error) {
+func serializeExport(export any, ar *api.Request) (data []byte, err error) {
-	// Serialize data.
+	// Get format.
-	data, mimeType, format, err := dsd.MimeDump(export, ar.Header.Get("Accept"))
+	format := dsd.FormatFromAccept(ar.Header.Get("Accept"))
-	if err != nil {
-		return nil, fmt.Errorf("failed to serialize data: %w", err)
-	}
-	ar.ResponseHeader.Set("Content-Type", mimeType)
 
-	// Add checksum.
+	// Serialize and add checksum.
 	switch format {
 	case dsd.JSON:
-		data, err = filesig.AddJSONChecksum(data)
+		data, err = json.Marshal(export)
+		if err == nil {
+			data, err = filesig.AddJSONChecksum(data)
+		}
 	case dsd.YAML:
-		data, err = filesig.AddYAMLChecksum(data, filesig.TextPlacementBottom)
+		data, err = yaml.Marshal(export)
+		if err == nil {
+			data, err = filesig.AddYAMLChecksum(data, filesig.TextPlacementBottom)
+		}
 	default:
 		return nil, dsd.ErrIncompatibleFormat
 	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to serialize: %w", err)
+	}
+
+	// Set Content-Type HTTP Header.
+	ar.ResponseHeader.Set("Content-Type", dsd.FormatToMimeType[format])
+
+	return data, nil
+}
+
+func serializeProfileExport(export *ProfileExport, ar *api.Request) ([]byte, error) {
+	// Do a regular serialize, if we don't need parts.
+	switch {
+	case export.IconData == "":
+		// With no icon, do a regular export.
+		return serializeExport(export, ar)
+	case dsd.FormatFromAccept(ar.Header.Get("Accept")) != dsd.YAML:
+		// Only export in parts for yaml.
+		return serializeExport(export, ar)
+	}
+
+	// Step 1: Separate profile icon.
+	profileIconExport := &ProfileIcon{
+		IconData: export.IconData,
+	}
+	export.IconData = ""
+
+	// Step 2: Serialize main export.
+	profileData, err := yaml.Marshal(export)
+	if err != nil {
+		return nil, fmt.Errorf("failed to serialize profile data: %w", err)
+	}
+
+	// Step 3: Serialize icon only.
+	iconData, err := yaml.Marshal(profileIconExport)
+	if err != nil {
+		return nil, fmt.Errorf("failed to serialize profile icon: %w", err)
+	}
+
+	// Step 4: Stitch data together and add copyright notice for icon.
+	exportData := container.New(
+		profileData,
+		[]byte(`
+# The application icon below is the property of its respective owner.
+# The icon is used for identification purposes only, and does not imply any endorsement or affiliation with their respective owners.
+# It is the sole responsibility of the individual or entity sharing this dataset to ensure they have the necessary permissions to do so.
+`),
+		iconData,
+	).CompileData()
+
+	// Step 4: Add checksum.
+	exportData, err = filesig.AddYAMLChecksum(exportData, filesig.TextPlacementBottom)
 	if err != nil {
 		return nil, fmt.Errorf("failed to add checksum: %w", err)
 	}
 
-	return data, nil
+	// Set Content-Type HTTP Header.
+	ar.ResponseHeader.Set("Content-Type", dsd.FormatToMimeType[dsd.YAML])
+
+	return exportData, nil
 }
 
 func parseExport(request *ImportRequest, export any) error {