Add first set of import/export APIs

This commit is contained in:
Daniel 2023-09-19 16:10:10 +02:00
parent 7c3925db32
commit cf3bd9f671
8 changed files with 743 additions and 14 deletions

View file

@ -13,6 +13,7 @@ import (
_ "github.com/safing/portmaster/netenv"
_ "github.com/safing/portmaster/netquery"
_ "github.com/safing/portmaster/status"
_ "github.com/safing/portmaster/sync"
_ "github.com/safing/portmaster/ui"
"github.com/safing/portmaster/updates"
)
@ -29,7 +30,7 @@ var (
)
func init() {
module = modules.Register("core", prep, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui", "netenv", "network", "netquery", "interception", "compat", "broadcasts")
module = modules.Register("core", prep, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui", "netenv", "network", "netquery", "interception", "compat", "broadcasts", "sync")
subsystems.Register(
"core",
"Core",

View file

@ -16,7 +16,7 @@ import (
// core:profiles/<scope>/<id>
// cache:profiles/index/<identifier>/<value>
const profilesDBPath = "core:profiles/"
const ProfilesDBPath = "core:profiles/"
var profileDB = database.NewInterface(&database.Options{
Local: true,
@ -28,17 +28,17 @@ func makeScopedID(source profileSource, id string) string {
}
func makeProfileKey(source profileSource, id string) string {
return profilesDBPath + string(source) + "/" + id
return ProfilesDBPath + string(source) + "/" + id
}
func registerValidationDBHook() (err error) {
_, err = database.RegisterHook(query.New(profilesDBPath), &databaseHook{})
_, err = database.RegisterHook(query.New(ProfilesDBPath), &databaseHook{})
return
}
func startProfileUpdateChecker() error {
module.StartServiceWorker("update active profiles", 0, func(ctx context.Context) (err error) {
profilesSub, err := profileDB.Subscribe(query.New(profilesDBPath))
profilesSub, err := profileDB.Subscribe(query.New(ProfilesDBPath))
if err != nil {
return err
}
@ -59,7 +59,7 @@ func startProfileUpdateChecker() error {
}
// Get active profile.
activeProfile := getActiveProfile(strings.TrimPrefix(r.Key(), profilesDBPath))
activeProfile := getActiveProfile(strings.TrimPrefix(r.Key(), ProfilesDBPath))
if activeProfile == nil {
// Don't do any additional actions if the profile is not active.
continue profileFeed

View file

@ -177,7 +177,7 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P
// getProfile fetches the profile for the given scoped ID.
func getProfile(scopedID string) (profile *Profile, err error) {
// Get profile from the database.
r, err := profileDB.Get(profilesDBPath + scopedID)
r, err := profileDB.Get(ProfilesDBPath + scopedID)
if err != nil {
return nil, err
}
@ -193,7 +193,7 @@ func findProfile(source profileSource, md MatchingData) (profile *Profile, err e
// process might be quite expensive. Measure impact and possibly improve.
// Get iterator over all profiles.
it, err := profileDB.Query(query.New(profilesDBPath + makeScopedID(source, "")))
it, err := profileDB.Query(query.New(ProfilesDBPath + makeScopedID(source, "")))
if err != nil {
return nil, fmt.Errorf("failed to query for profiles: %w", err)
}
@ -299,7 +299,7 @@ func notifyConflictingProfiles(a, b record.Record, md MatchingData) {
idA = profileA.ScopedID()
nameA = profileA.Name
} else {
idA = strings.TrimPrefix(a.Key(), profilesDBPath)
idA = strings.TrimPrefix(a.Key(), ProfilesDBPath)
nameA = path.Base(idA)
}
profileB, err := EnsureProfile(b)
@ -307,7 +307,7 @@ func notifyConflictingProfiles(a, b record.Record, md MatchingData) {
idB = profileB.ScopedID()
nameB = profileB.Name
} else {
idB = strings.TrimPrefix(b.Key(), profilesDBPath)
idB = strings.TrimPrefix(b.Key(), ProfilesDBPath)
nameB = path.Base(idB)
}

View file

@ -71,7 +71,7 @@ func migrateNetworkRatingSystem(ctx context.Context, _, to *version.Version, db
func migrateLinkedPath(ctx context.Context, _, to *version.Version, db *database.Interface) error {
// Get iterator over all profiles.
it, err := db.Query(query.New(profilesDBPath))
it, err := db.Query(query.New(ProfilesDBPath))
if err != nil {
log.Tracer(ctx).Errorf("profile: failed to migrate from linked path: failed to start query: %s", err)
return nil
@ -112,7 +112,7 @@ func migrateLinkedPath(ctx context.Context, _, to *version.Version, db *database
func migrateIcons(ctx context.Context, _, to *version.Version, db *database.Interface) error {
// Get iterator over all profiles.
it, err := db.Query(query.New(profilesDBPath))
it, err := db.Query(query.New(ProfilesDBPath))
if err != nil {
log.Tracer(ctx).Errorf("profile: failed to migrate from icon fields: failed to start query: %s", err)
return nil
@ -181,7 +181,7 @@ func migrateToDerivedIDs(ctx context.Context, _, to *version.Version, db *databa
var profilesToDelete []string //nolint:prealloc // We don't know how many profiles there are.
// Get iterator over all profiles.
it, err := db.Query(query.New(profilesDBPath))
it, err := db.Query(query.New(ProfilesDBPath))
if err != nil {
log.Tracer(ctx).Errorf("profile: failed to migrate to derived profile IDs: failed to start query: %s", err)
return nil
@ -243,7 +243,7 @@ func migrateToDerivedIDs(ctx context.Context, _, to *version.Version, db *databa
// Delete old migrated profiles.
for _, scopedID := range profilesToDelete {
if err := db.Delete(profilesDBPath + scopedID); err != nil {
if err := db.Delete(ProfilesDBPath + scopedID); err != nil {
log.Tracer(ctx).Errorf("profile: failed to delete old profile %s during migration: %s", scopedID, err)
}
}

114
sync/module.go Normal file
View file

@ -0,0 +1,114 @@
package sync
import (
"errors"
"net/http"
"github.com/safing/portbase/api"
"github.com/safing/portbase/database"
"github.com/safing/portbase/modules"
)
var (
module *modules.Module
db = database.NewInterface(&database.Options{
Local: true,
Internal: true,
})
)
func init() {
module = modules.Register("sync", prep, nil, nil, "profiles")
}
func prep() error {
if err := registerSettingsAPI(); err != nil {
return err
}
if err := registerSingleSettingAPI(); err != nil {
return err
}
return nil
}
// Type is the type of an export.
type Type string
// Export Types.
const (
TypeProfile = "profile"
TypeSettings = "settings"
TypeSingleSetting = "single-setting"
)
// Export IDs.
const (
ExportTargetGlobal = "global"
)
// Messages.
var (
MsgNone = ""
MsgValid = "Import is valid."
MsgSuccess = "Import successful."
MsgRequireRestart = "Import successful. Restart required for setting to take effect."
)
// ExportRequest is a request for an export.
type ExportRequest struct {
From string `json:"from"`
Key string `json:"key"`
}
// ImportRequest is a request to import an export.
type ImportRequest struct {
// Where the export should be import to.
Target string `json:"target"`
// Only validate, but do not actually change anything.
ValidateOnly bool `json:"validate_only"`
RawExport string `json:"raw_export"`
}
// ImportResult is returned by successful import operations.
type ImportResult struct {
RestartRequired bool `json:"restart_required"`
ReplacesExisting bool `json:"replaces_existing"`
}
// Errors.
var (
ErrMismatch = api.ErrorWithStatus(
errors.New("the supplied export cannot be imported here"),
http.StatusPreconditionFailed,
)
ErrTargetNotFound = api.ErrorWithStatus(
errors.New("import/export target does not exist"),
http.StatusGone,
)
ErrUnchanged = api.ErrorWithStatus(
errors.New("cannot export unchanged setting"),
http.StatusGone,
)
ErrInvalidImport = api.ErrorWithStatus(
errors.New("invalid import"),
http.StatusUnprocessableEntity,
)
ErrInvalidSetting = api.ErrorWithStatus(
errors.New("invalid setting"),
http.StatusUnprocessableEntity,
)
ErrInvalidProfile = api.ErrorWithStatus(
errors.New("invalid profile"),
http.StatusUnprocessableEntity,
)
ErrImportFailed = api.ErrorWithStatus(
errors.New("import failed"),
http.StatusInternalServerError,
)
ErrExportFailed = api.ErrorWithStatus(
errors.New("export failed"),
http.StatusInternalServerError,
)
)

45
sync/profile.go Normal file
View file

@ -0,0 +1,45 @@
package sync
import (
"time"
"github.com/safing/portmaster/profile"
)
// ProfileExport holds an export of a profile.
type ProfileExport struct { //nolint:maligned
Type Type
// Identification (sync or import as new only)
ID string
Source string
// Human Metadata
Name string
Description string
Homepage string
Icons []profile.Icon
PresentationPath string
UsePresentationPath bool
// Process matching
Fingerprints []profile.Fingerprint
// Settings
Config map[string]any
// Metadata (sync only)
LastEdited time.Time
Created time.Time
Internal bool
}
// ProfileImportRequest is a request to import Profile.
type ProfileImportRequest struct {
ImportRequest
// Reset all settings and fingerprints of target before import.
Reset bool
Export *ProfileExport
}

264
sync/setting_single.go Normal file
View file

@ -0,0 +1,264 @@
package sync
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/ghodss/yaml"
"github.com/safing/portbase/api"
"github.com/safing/portbase/config"
"github.com/safing/portmaster/profile"
)
// SingleSettingExport holds an export of a single setting.
type SingleSettingExport struct {
Type Type `json:"type"` // Must be TypeSingleSetting
ID string `json:"id"` // Settings Key
Value any `json:"value"`
}
// SingleSettingImportRequest is a request to import a single setting.
type SingleSettingImportRequest struct {
ImportRequest `json:",inline"`
Export *SingleSettingExport `json:"export"`
}
func registerSingleSettingAPI() error {
if err := api.RegisterEndpoint(api.Endpoint{
Name: "Export Single Setting",
Description: "Exports a single setting in a share-able format.",
Path: "sync/single-setting/export",
Read: api.PermitAdmin,
Write: api.PermitAdmin,
Parameters: []api.Parameter{{
Method: http.MethodGet,
Field: "from",
Description: "Specify where to export from.",
}, {
Method: http.MethodGet,
Field: "key",
Description: "Specify which settings key to export.",
}},
BelongsTo: module,
DataFunc: handleExportSingleSetting,
}); err != nil {
return err
}
if err := api.RegisterEndpoint(api.Endpoint{
Name: "Import Single Setting",
Description: "Imports a single setting from the share-able format.",
Path: "sync/single-setting/import",
Read: api.PermitAdmin,
Write: api.PermitAdmin,
Parameters: []api.Parameter{{
Method: http.MethodPost,
Field: "to",
Description: "Specify where to import to.",
}, {
Method: http.MethodPost,
Field: "key",
Description: "Specify which setting key to import.",
}, {
Method: http.MethodPost,
Field: "validate",
Description: "Validate only.",
}},
BelongsTo: module,
StructFunc: handleImportSingleSetting,
}); err != nil {
return err
}
return nil
}
func handleExportSingleSetting(ar *api.Request) (data []byte, err error) {
var request *ExportRequest
// Get parameters.
q := ar.URL.Query()
if len(q) > 0 {
request = &ExportRequest{
From: q.Get("from"),
Key: q.Get("key"),
}
} else {
request = &ExportRequest{}
if err := json.Unmarshal(ar.InputData, request); err != nil {
return nil, fmt.Errorf("%w: failed to parse export request: %s", ErrExportFailed, err)
}
}
// Check parameters.
if request.From == "" || request.Key == "" {
return nil, errors.New("missing parameters")
}
// Export.
export, err := ExportSingleSetting(request.Key, request.From)
if err != nil {
return nil, err
}
// Make some yummy yaml.
yamlData, err := yaml.Marshal(export)
if err != nil {
return nil, fmt.Errorf("%w: failed to marshal to yaml: %s", ErrExportFailed, err)
}
// TODO: Add checksum for integrity.
return yamlData, nil
}
func handleImportSingleSetting(ar *api.Request) (any, error) {
var request *SingleSettingImportRequest
// Get parameters.
q := ar.URL.Query()
if len(q) > 0 {
request = &SingleSettingImportRequest{
ImportRequest: ImportRequest{
Target: q.Get("to"),
ValidateOnly: q.Has("validate"),
RawExport: string(ar.InputData),
},
}
} else {
request = &SingleSettingImportRequest{}
if err := json.Unmarshal(ar.InputData, request); err != nil {
return nil, fmt.Errorf("%w: failed to parse import request: %s", ErrInvalidImport, err)
}
}
// Check if we need to parse the export.
switch {
case request.Export != nil && request.RawExport != "":
return nil, fmt.Errorf("%w: both Export and RawExport are defined", ErrInvalidImport)
case request.RawExport != "":
// TODO: Verify checksum for integrity.
export := &SingleSettingExport{}
if err := yaml.Unmarshal([]byte(request.RawExport), export); err != nil {
return nil, fmt.Errorf("%w: failed to parse export: %s", ErrInvalidImport, err)
}
request.Export = export
}
// Optional check if the setting key matches.
if q.Has("key") && q.Get("key") != request.Export.ID {
return nil, ErrMismatch
}
// Import.
return ImportSingeSetting(request)
}
// ExportSingleSetting export a single setting.
func ExportSingleSetting(key, from string) (*SingleSettingExport, error) {
var value any
if from == ExportTargetGlobal {
option, err := config.GetOption(key)
if err != nil {
return nil, fmt.Errorf("%w: configuration %s", ErrTargetNotFound, err)
}
value = option.UserValue()
if value == nil {
return nil, ErrUnchanged
}
} else {
r, err := db.Get(profile.ProfilesDBPath + from)
if err != nil {
return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
}
p, err := profile.EnsureProfile(r)
if err != nil {
return nil, fmt.Errorf("%w: failed to load profile: %s", ErrExportFailed, err)
}
flattened := config.Flatten(p.Config)
value = flattened[key]
if value == nil {
return nil, ErrUnchanged
}
}
return &SingleSettingExport{
Type: TypeSingleSetting,
ID: key,
Value: value,
}, nil
}
// ImportSingeSetting imports a single setting.
func ImportSingeSetting(r *SingleSettingImportRequest) (*ImportResult, error) {
// Check import.
if r.Export.Type != TypeSingleSetting {
return nil, ErrMismatch
}
// Get option and validate value.
option, err := config.GetOption(r.Export.ID)
if err != nil {
return nil, fmt.Errorf("%w: configuration %s", ErrTargetNotFound, err)
}
if option.ValidateValue(r.Export.Value) != nil {
return nil, fmt.Errorf("%w: configuration value is invalid: %s", ErrInvalidSetting, err)
}
// Import single global setting.
if r.Target == ExportTargetGlobal {
// Stop here if we are only validating.
if r.ValidateOnly {
return &ImportResult{
RestartRequired: option.RequiresRestart,
ReplacesExisting: option.IsSetByUser(),
}, nil
}
// Actually import the setting.
err = config.SetConfigOption(r.Export.ID, r.Export.Value)
if err != nil {
return nil, fmt.Errorf("%w: configuration value is invalid: %s", ErrInvalidSetting, err)
}
} else {
// Import single setting into profile.
rec, err := db.Get(profile.ProfilesDBPath + r.Target)
if err != nil {
return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
}
p, err := profile.EnsureProfile(rec)
if err != nil {
return nil, fmt.Errorf("%w: failed to load profile: %s", ErrImportFailed, err)
}
// Stop here if we are only validating.
if r.ValidateOnly {
return &ImportResult{
RestartRequired: option.RequiresRestart,
ReplacesExisting: option.IsSetByUser(),
}, nil
}
// Set imported setting on profile.
flattened := config.Flatten(p.Config)
flattened[r.Export.ID] = r.Export.Value
p.Config = config.Expand(flattened)
// Save profile back to db.
err = p.Save()
if err != nil {
return nil, fmt.Errorf("%w: failed to save profile: %s", ErrImportFailed, err)
}
}
return &ImportResult{
RestartRequired: option.RequiresRestart,
ReplacesExisting: option.IsSetByUser(),
}, nil
}

305
sync/settings.go Normal file
View file

@ -0,0 +1,305 @@
package sync
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"github.com/ghodss/yaml"
"github.com/safing/portbase/api"
"github.com/safing/portbase/config"
"github.com/safing/portmaster/profile"
)
// SettingsExport holds an export of settings.
type SettingsExport struct {
Type Type `json:"type"`
Config map[string]any `json:"config"`
}
// SettingsImportRequest is a request to import settings.
type SettingsImportRequest struct {
ImportRequest `json:",inline"`
// Reset all settings of target before import.
// The ImportResult also reacts to this flag and correctly reports whether
// any settings would be replaced or deleted.
Reset bool `json:"reset"`
Export *SettingsExport `json:"export"`
}
func registerSettingsAPI() error {
if err := api.RegisterEndpoint(api.Endpoint{
Name: "Export Settings",
Description: "Exports settings in a share-able format.",
Path: "sync/settings/export",
Read: api.PermitAdmin,
Write: api.PermitAdmin,
Parameters: []api.Parameter{{
Method: http.MethodGet,
Field: "from",
Description: "Specify where to export from.",
}},
BelongsTo: module,
DataFunc: handleExportSettings,
}); err != nil {
return err
}
if err := api.RegisterEndpoint(api.Endpoint{
Name: "Import Settings",
Description: "Imports settings from the share-able format.",
Path: "sync/settings/import",
Read: api.PermitAdmin,
Write: api.PermitAdmin,
Parameters: []api.Parameter{{
Method: http.MethodPost,
Field: "to",
Description: "Specify where to import to.",
}, {
Method: http.MethodPost,
Field: "validate",
Description: "Validate only.",
}, {
Method: http.MethodPost,
Field: "reset",
Description: "Replace all existing settings.",
}},
BelongsTo: module,
StructFunc: handleImportSettings,
}); err != nil {
return err
}
return nil
}
func handleExportSettings(ar *api.Request) (data []byte, err error) {
var request *ExportRequest
// Get parameters.
q := ar.URL.Query()
if len(q) > 0 {
request = &ExportRequest{
From: q.Get("from"),
}
} else {
request = &ExportRequest{}
if err := json.Unmarshal(ar.InputData, request); err != nil {
return nil, fmt.Errorf("%w: failed to parse export request: %s", ErrExportFailed, err)
}
}
// Check parameters.
if request.From == "" {
return nil, errors.New("missing parameters")
}
// Export.
export, err := ExportSettings(request.From)
if err != nil {
return nil, err
}
// Make some yummy yaml.
yamlData, err := yaml.Marshal(export)
if err != nil {
return nil, fmt.Errorf("%w: failed to marshal to yaml: %s", ErrExportFailed, err)
}
// TODO: Add checksum for integrity.
return yamlData, nil
}
func handleImportSettings(ar *api.Request) (any, error) {
var request *SettingsImportRequest
// Get parameters.
q := ar.URL.Query()
if len(q) > 0 {
request = &SettingsImportRequest{
ImportRequest: ImportRequest{
Target: q.Get("to"),
ValidateOnly: q.Has("validate"),
RawExport: string(ar.InputData),
},
Reset: q.Has("reset"),
}
} else {
request = &SettingsImportRequest{}
if err := json.Unmarshal(ar.InputData, request); err != nil {
return nil, fmt.Errorf("%w: failed to parse import request: %s", ErrInvalidImport, err)
}
}
// Check if we need to parse the export.
switch {
case request.Export != nil && request.RawExport != "":
return nil, fmt.Errorf("%w: both Export and RawExport are defined", ErrInvalidImport)
case request.RawExport != "":
// TODO: Verify checksum for integrity.
export := &SettingsExport{}
if err := yaml.Unmarshal([]byte(request.RawExport), export); err != nil {
return nil, fmt.Errorf("%w: failed to parse export: %s", ErrInvalidImport, err)
}
request.Export = export
}
// Import.
return ImportSettings(request)
}
// ExportSettings exports the global settings.
func ExportSettings(from string) (*SettingsExport, error) {
var settings map[string]any
if from == ExportTargetGlobal {
// Collect all changed global settings.
settings = make(map[string]any)
_ = config.ForEachOption(func(option *config.Option) error {
v := option.UserValue()
if v != nil {
settings[option.Key] = v
}
return nil
})
} else {
r, err := db.Get(profile.ProfilesDBPath + from)
if err != nil {
return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
}
p, err := profile.EnsureProfile(r)
if err != nil {
return nil, fmt.Errorf("%w: failed to load profile: %s", ErrExportFailed, err)
}
settings = config.Flatten(p.Config)
}
// Check if there any changed settings.
if len(settings) == 0 {
return nil, ErrUnchanged
}
// Expand config to hierarchical form.
settings = config.Expand(settings)
return &SettingsExport{
Type: TypeSettings,
Config: settings,
}, nil
}
// ImportSettings imports the global settings.
func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) {
// Check import.
if r.Export.Type != TypeSettings {
return nil, ErrMismatch
}
// Flatten config.
settings := config.Flatten(r.Export.Config)
// Validate config and gather some metadata.
var (
result = &ImportResult{}
checked int
)
err := config.ForEachOption(func(option *config.Option) error {
// Check if any setting is set.
if r.Reset && option.IsSetByUser() {
result.ReplacesExisting = true
}
newValue, ok := settings[option.Key]
if ok {
checked++
// Validate the new value.
if err := option.ValidateValue(newValue); err != nil {
return fmt.Errorf("%w: configuration value for %s is invalid: %s", ErrInvalidSetting, option.Key, err)
}
// Collect metadata.
if option.RequiresRestart {
result.RestartRequired = true
}
if !r.Reset && option.IsSetByUser() {
result.ReplacesExisting = true
}
}
return nil
})
if err != nil {
return nil, err
}
if checked < len(settings) {
return nil, fmt.Errorf("%w: the export contains unknown settings", ErrInvalidImport)
}
// Import global settings.
if r.Target == ExportTargetGlobal {
// Stop here if we are only validating.
if r.ValidateOnly {
return result, nil
}
// Import to global config.
vErrs, restartRequired := config.ReplaceConfig(settings)
if len(vErrs) > 0 {
s := make([]string, 0, len(vErrs))
for _, err := range vErrs {
s = append(s, err.Error())
}
return nil, fmt.Errorf(
"%w: the supplied configuration could not be applied:\n%s",
ErrImportFailed,
strings.Join(s, "\n"),
)
}
result.RestartRequired = restartRequired
return result, nil
}
// Import settings into profile.
rec, err := db.Get(profile.ProfilesDBPath + r.Target)
if err != nil {
return nil, fmt.Errorf("%w: failed to find profile: %s", ErrTargetNotFound, err)
}
p, err := profile.EnsureProfile(rec)
if err != nil {
return nil, fmt.Errorf("%w: failed to load profile: %s", ErrImportFailed, err)
}
// FIXME: check if there are any global-only setting in the import
// Stop here if we are only validating.
if r.ValidateOnly {
return result, nil
}
// Import settings into profile.
if r.Reset {
p.Config = config.Expand(settings)
} else {
flattenedProfileConfig := config.Flatten(p.Config)
for k, v := range settings {
flattenedProfileConfig[k] = v
}
p.Config = config.Expand(flattenedProfileConfig)
}
// Save profile back to db.
err = p.Save()
if err != nil {
return nil, fmt.Errorf("%w: failed to save profile: %s", ErrImportFailed, err)
}
return result, nil
}