From 480807a31cfec6d7498447b79d82b9c8e0404839 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 5 May 2021 00:11:10 +0200 Subject: [PATCH 01/16] Improve flags --- api/config.go | 14 ++++++-------- api/main.go | 1 - config/basic_config.go | 2 +- metrics/config.go | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/api/config.go b/api/config.go index ebcc515..354ea2a 100644 --- a/api/config.go +++ b/api/config.go @@ -4,7 +4,6 @@ import ( "flag" "github.com/safing/portbase/config" - "github.com/safing/portbase/log" ) // Config Keys. @@ -24,13 +23,12 @@ var ( ) func init() { - flag.StringVar(&listenAddressFlag, "api-address", "", "override api listen address") -} - -func logFlagOverrides() { - if listenAddressFlag != "" { - log.Warning("api: api/listenAddress default config is being overridden by -api-address flag") - } + flag.StringVar( + &listenAddressFlag, + "api-address", + "", + "set api listen address; configuration is stronger", + ) } func getDefaultListenAddress() string { diff --git a/api/main.go b/api/main.go index e47b618..17e2a6d 100644 --- a/api/main.go +++ b/api/main.go @@ -54,7 +54,6 @@ func prep() error { } func start() error { - logFlagOverrides() go Serve() _ = updateAPIKeys(module.Ctx, nil) diff --git a/config/basic_config.go b/config/basic_config.go index ce4a716..6975231 100644 --- a/config/basic_config.go +++ b/config/basic_config.go @@ -18,7 +18,7 @@ var ( ) func init() { - flag.BoolVar(&defaultDevMode, "devmode", false, "enable development mode") + flag.BoolVar(&defaultDevMode, "devmode", false, "enable development mode; configuration is stronger") } func registerBasicOptions() error { diff --git a/metrics/config.go b/metrics/config.go index 9767d76..dfcfc02 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -21,8 +21,8 @@ var ( ) func init() { - flag.StringVar(&pushFlag, "push-metrics", "", "Set default URL to push prometheus metrics to.") - flag.StringVar(&instanceFlag, "metrics-instance", "", "Set the default global instance label.") + flag.StringVar(&pushFlag, "push-metrics", "", "set default URL to push prometheus metrics to") + flag.StringVar(&instanceFlag, "metrics-instance", "", "set the default global instance label") } func prepConfig() error { From ca19f4a44bfa881add5348bbd2ac5199896219e7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 5 May 2021 00:12:43 +0200 Subject: [PATCH 02/16] Improve notifications and module failures and mirror them --- modules/modules.go | 20 +++-- modules/status.go | 85 +++++++++++++++---- modules/subsystems/module.go | 3 +- modules/subsystems/subsystems_test.go | 2 +- notifications/module-mirror.go | 95 +++++++++++++++++++++ notifications/notification.go | 117 +++++++++++++++++++++----- 6 files changed, 275 insertions(+), 47 deletions(-) create mode 100644 notifications/module-mirror.go diff --git a/modules/modules.go b/modules/modules.go index 286e3e2..2437a1e 100644 --- a/modules/modules.go +++ b/modules/modules.go @@ -40,6 +40,7 @@ type Module struct { //nolint:maligned // not worth the effort // failure status failureStatus uint8 failureID string + failureTitle string failureMsg string // lifecycle callback functions @@ -62,7 +63,7 @@ type Module struct { //nolint:maligned // not worth the effort waitGroup sync.WaitGroup // events - eventHooks map[string][]*eventHook + eventHooks map[string]*eventHooks eventHooksLock sync.RWMutex // dependency mgmt @@ -127,8 +128,9 @@ func (m *Module) prep(reports chan *report) { // set status if err != nil { m.Error( - "module-failed-prep", - fmt.Sprintf("failed to prep module: %s", err.Error()), + fmt.Sprintf("%s:prep-failed", m.Name), + fmt.Sprintf("Preparing module %s failed", m.Name), + fmt.Sprintf("Failed to prep module: %s", err.Error()), ) } else { m.Lock() @@ -183,8 +185,9 @@ func (m *Module) start(reports chan *report) { // set status if err != nil { m.Error( - "module-failed-start", - fmt.Sprintf("failed to start module: %s", err.Error()), + fmt.Sprintf("%s:start-failed", m.Name), + fmt.Sprintf("Starting module %s failed", m.Name), + fmt.Sprintf("Failed to start module: %s", err.Error()), ) } else { m.Lock() @@ -270,8 +273,9 @@ func (m *Module) stopAllTasks(reports chan *report) { // set status if err != nil { m.Error( - "module-failed-stop", - fmt.Sprintf("failed to stop module: %s", err.Error()), + fmt.Sprintf("%s:stop-failed", m.Name), + fmt.Sprintf("Stopping module %s failed", m.Name), + fmt.Sprintf("Failed to stop module: %s", err.Error()), ) } @@ -328,7 +332,7 @@ func initNewModule(name string, prep, start, stop func() error, dependencies ... taskCnt: &taskCnt, microTaskCnt: µTaskCnt, waitGroup: sync.WaitGroup{}, - eventHooks: make(map[string][]*eventHook), + eventHooks: make(map[string]*eventHooks), depNames: dependencies, } diff --git a/modules/status.go b/modules/status.go index 94ccc1b..3cc727a 100644 --- a/modules/status.go +++ b/modules/status.go @@ -1,5 +1,11 @@ package modules +import ( + "context" + + "github.com/tevino/abool" +) + // Module Status Values const ( StatusDead uint8 = 0 // not prepared, not started @@ -25,6 +31,23 @@ const ( statusNothingToDo ) +var ( + failureUpdateNotifyFunc func(moduleFailure uint8, id, title, msg string) + failureUpdateNotifyFuncEnabled = abool.NewBool(false) + failureUpdateNotifyFuncReady = abool.NewBool(false) +) + +// SetFailureUpdateNotifyFunc sets a function that is called on every change +// of a module's failure status. +func SetFailureUpdateNotifyFunc(fn func(moduleFailure uint8, id, title, msg string)) bool { + if failureUpdateNotifyFuncEnabled.SetToIf(false, true) { + failureUpdateNotifyFunc = fn + failureUpdateNotifyFuncReady.Set() + return true + } + return false +} + // Online returns whether the module is online. func (m *Module) Online() bool { return m.Status() == StatusOnline @@ -57,38 +80,52 @@ func (m *Module) FailureStatus() (failureStatus uint8, failureID, failureMsg str } // Hint sets failure status to hint. This is a somewhat special failure status, as the module is believed to be working correctly, but there is an important module specific information to convey. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. -func (m *Module) Hint(failureID, failureMsg string) { +func (m *Module) Hint(id, title, msg string) { m.Lock() defer m.Unlock() - m.failureStatus = FailureHint - m.failureID = failureID - m.failureMsg = failureMsg - - m.notifyOfChange() + m.setFailure(FailureHint, id, title, msg) } // Warning sets failure status to warning. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. -func (m *Module) Warning(failureID, failureMsg string) { +func (m *Module) Warning(id, title, msg string) { m.Lock() defer m.Unlock() - m.failureStatus = FailureWarning - m.failureID = failureID - m.failureMsg = failureMsg - - m.notifyOfChange() + m.setFailure(FailureWarning, id, title, msg) } // Error sets failure status to error. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. -func (m *Module) Error(failureID, failureMsg string) { +func (m *Module) Error(id, title, msg string) { m.Lock() defer m.Unlock() - m.failureStatus = FailureError - m.failureID = failureID - m.failureMsg = failureMsg + m.setFailure(FailureError, id, title, msg) +} +func (m *Module) setFailure(status uint8, id, title, msg string) { + // Send an update before we override a previous failure. + if failureUpdateNotifyFuncReady.IsSet() && m.failureID != "" { + updateFailureID := m.failureID + m.StartWorker("failure status updater", func(context.Context) error { + // Only use data in worker that won't change anymore. + failureUpdateNotifyFunc(FailureNone, updateFailureID, "", "") + return nil + }) + } + + m.failureStatus = status + m.failureID = id + m.failureTitle = title + m.failureMsg = msg + + if failureUpdateNotifyFuncReady.IsSet() { + m.StartWorker("failure status updater", func(context.Context) error { + // Only use data in worker that won't change anymore. + failureUpdateNotifyFunc(status, id, title, msg) + return nil + }) + } m.notifyOfChange() } @@ -98,12 +135,24 @@ func (m *Module) Resolve(failureID string) { defer m.Unlock() if failureID == "" || failureID == m.failureID { + // Propagate resolving. + if failureUpdateNotifyFuncReady.IsSet() { + updateFailureID := m.failureID + m.StartWorker("failure status updater", func(context.Context) error { + // Only use data in worker that won't change anymore. + failureUpdateNotifyFunc(FailureNone, updateFailureID, "", "") + return nil + }) + } + + // Set failure status on module. m.failureStatus = FailureNone m.failureID = "" + m.failureTitle = "" m.failureMsg = "" - } - m.notifyOfChange() + m.notifyOfChange() + } } // readyToPrep returns whether all dependencies are ready for this module to prep. diff --git a/modules/subsystems/module.go b/modules/subsystems/module.go index 539fe77..48a3784 100644 --- a/modules/subsystems/module.go +++ b/modules/subsystems/module.go @@ -76,7 +76,8 @@ func prep() error { if err != nil { module.Error( "modulemgmt-failed", - fmt.Sprintf("The subsystem framework failed to start or stop one or more modules.\nError: %s\nCheck logs for more information.", err), + "A Module failed to start", + fmt.Sprintf("The subsystem framework failed to start or stop one or more modules.\nError: %s\nCheck logs for more information or try to restart.", err), ) return nil } diff --git a/modules/subsystems/subsystems_test.go b/modules/subsystems/subsystems_test.go index 3333df6..4351a34 100644 --- a/modules/subsystems/subsystems_test.go +++ b/modules/subsystems/subsystems_test.go @@ -77,7 +77,7 @@ func TestSubsystems(t *testing.T) { // test // let module fail - feature1.Error("test-fail", "Testing Fail") + feature1.Error("test-fail", "Test Fail", "Testing Fail") time.Sleep(10 * time.Millisecond) if sub1.FailureStatus != modules.FailureError { t.Fatal("error did not propagate") diff --git a/notifications/module-mirror.go b/notifications/module-mirror.go new file mode 100644 index 0000000..5d0fd2f --- /dev/null +++ b/notifications/module-mirror.go @@ -0,0 +1,95 @@ +package notifications + +import ( + "github.com/safing/portbase/log" + "github.com/safing/portbase/modules" +) + +// AttachToModule attaches the notification to a module and changes to the +// notification will be reflected on the module failure status. +func (n *Notification) AttachToModule(m *modules.Module) { + log.Errorf("notifications: attaching %q", n.EventID) + + if m == nil { + log.Warningf("notifications: cannot remove attached module from notification %s", n.EventID) + return + } + + n.lock.Lock() + defer n.lock.Unlock() + + if n.State != Active { + log.Warningf("notifications: cannot attach module to inactive notification %s", n.EventID) + return + } + if n.belongsTo != nil { + log.Warningf("notifications: cannot override attached module for notification %s", n.EventID) + return + } + + // Attach module. + n.belongsTo = m + + // Set module failure status. + switch n.Type { //nolint:exhaustive + case Info: + m.Hint(n.EventID, n.Title, n.Message) + case Warning: + m.Warning(n.EventID, n.Title, n.Message) + case Error: + m.Error(n.EventID, n.Title, n.Message) + default: + log.Warningf("notifications: incompatible type for attaching to module in notification %s", n.EventID) + m.Error(n.EventID, n.Title, n.Message+" [incompatible notification type]") + } +} + +// resolveModuleFailure removes the notification from the module failure status. +func (n *Notification) resolveModuleFailure() { + log.Errorf("notifications: resolving %q", n.EventID) + + if n.belongsTo != nil { + // Resolve failure in attached module. + n.belongsTo.Resolve(n.EventID) + + // Reset attachment in order to mitigate duplicate failure resolving. + // Re-attachment is prevented by the state check when attaching. + n.belongsTo = nil + } +} + +func init() { + modules.SetFailureUpdateNotifyFunc(mirrorModuleStatus) +} + +func mirrorModuleStatus(moduleFailure uint8, id, title, msg string) { + log.Errorf("notifications: mirroring %d %q %q %q", moduleFailure, id, title, msg) + + // Ignore "resolve all" requests. + if id == "" { + return + } + + // Get notification from storage. + n, ok := getNotification(id) + if ok { + // The notification already exists. + + // Check if we should delete it. + if moduleFailure == modules.FailureNone { + n.Delete() + } + + return + } + + // A notification for the given ID does not yet exists, create it. + switch moduleFailure { + case modules.FailureHint: + NotifyInfo(id, title, msg) + case modules.FailureWarning: + NotifyWarn(id, title, msg) + case modules.FailureError: + NotifyError(id, title, msg) + } +} diff --git a/notifications/notification.go b/notifications/notification.go index 0c2a6be..6e6615c 100644 --- a/notifications/notification.go +++ b/notifications/notification.go @@ -8,6 +8,7 @@ import ( "github.com/safing/portbase/database/record" "github.com/safing/portbase/log" + "github.com/safing/portbase/modules" "github.com/safing/portbase/utils" ) @@ -19,6 +20,7 @@ const ( Info Type = 0 Warning Type = 1 Prompt Type = 2 + Error Type = 3 ) // State describes the state of a notification. @@ -70,6 +72,9 @@ type Notification struct { // of the notification is available. Note that the message should already // have any paramerized values replaced. Message string + // ShowOnSystem specifies if the notification should be also shown on the + // operating system. + ShowOnSystem bool // EventData contains an additional payload for the notification. This payload // may contain contextual data and may be used by a localization framework // to populate the notification message template. @@ -91,6 +96,10 @@ type Notification struct { // based on the user selection. SelectedActionID string + // belongsTo holds the module this notification belongs to. The notification + // lifecycle will be mirrored to the module's failure status. + belongsTo *modules.Module + lock sync.Mutex actionFunction NotificationActionFn // call function to process action actionTrigger chan string // and/or send to a channel @@ -99,8 +108,53 @@ type Notification struct { // Action describes an action that can be taken for a notification. type Action struct { - ID string - Text string + ID string + Text string + Type ActionType + Payload interface{} + + // Dismisses specifies if the notification is dismissed when this action is selected. + Dismisses bool +} + +// ActionType defines a specific type of action. +type ActionType string + +// Action Types. +const ( + ActionTypeNone = "" // Report selected ID back to backend. + ActionTypeOpenURL = "open-url" // Open external URL + ActionTypeOpenPage = "open-page" // Payload: Page ID + ActionTypeOpenSetting = "open-setting" // Payload: See struct definition below. + ActionTypeOpenProfile = "open-profile" // Payload: Scoped Profile ID + ActionTypeInjectEvent = "inject-event" // Payload: Event ID + ActionTypeWebhook = "call-webhook" // Payload: See struct definition below. +) + +// ActionTypeOpenSettingPayload defines the payload for the OpenSetting Action Type. +type ActionTypeOpenSettingPayload struct { + // Key is the key of the setting. + Key string + // Profile is the scoped ID of the profile. + // Leaving this empty opens the global settings. + Profile string +} + +// ActionTypeWebhookPayload defines the payload for the WebhookPayload Action Type. +type ActionTypeWebhookPayload struct { + // HTTP Method to use. Defaults to "GET", or "POST" if a Payload is supplied. + Method string + // URL to call. + // If the URL is relative, prepend the current API endpoint base path. + // If the URL is absolute, send request to the Portmaster. + URL string + // Payload holds arbitrary payload data. + Payload interface{} + // ResultAction defines what should be done with successfully returned data. + // Must one of: + // - `ignore`: do nothing (default) + // - `display`: the result is a human readable message, display it in a success message. + ResultAction string } // Get returns the notification identifed by the given id or nil if it doesn't exist. @@ -114,28 +168,55 @@ func Get(id string) *Notification { return nil } +// Delete deletes the notification with the given id. +func Delete(id string) { + // Delete notification in defer to enable deferred unlocking. + var n *Notification + var ok bool + defer func() { + if ok { + n.Delete() + } + }() + + notsLock.Lock() + defer notsLock.Unlock() + n, ok = nots[id] +} + // NotifyInfo is a helper method for quickly showing a info // notification. The notification is already shown. If id is // an empty string a new UUIDv4 will be generated. -func NotifyInfo(id, msg string, actions ...Action) *Notification { - return notify(Info, id, msg, actions...) +// ShowOnSystem is disabled. +func NotifyInfo(id, title, msg string, actions ...Action) *Notification { + return notify(Info, id, title, msg, false, actions...) } // NotifyWarn is a helper method for quickly showing a warning // notification. The notification is already shown. If id is // an empty string a new UUIDv4 will be generated. -func NotifyWarn(id, msg string, actions ...Action) *Notification { - return notify(Warning, id, msg, actions...) +// ShowOnSystem is enabled. +func NotifyWarn(id, title, msg string, actions ...Action) *Notification { + return notify(Warning, id, title, msg, true, actions...) +} + +// NotifyError is a helper method for quickly showing an error +// notification. The notification is already shown. If id is +// an empty string a new UUIDv4 will be generated. +// ShowOnSystem is enabled. +func NotifyError(id, title, msg string, actions ...Action) *Notification { + return notify(Error, id, title, msg, true, actions...) } // NotifyPrompt is a helper method for quickly showing a prompt // notification. The notification is already shown. If id is // an empty string a new UUIDv4 will be generated. -func NotifyPrompt(id, msg string, actions ...Action) *Notification { - return notify(Prompt, id, msg, actions...) +// ShowOnSystem is disabled. +func NotifyPrompt(id, title, msg string, actions ...Action) *Notification { + return notify(Prompt, id, title, msg, false, actions...) } -func notify(nType Type, id, msg string, actions ...Action) *Notification { +func notify(nType Type, id, title, msg string, showOnSystem bool, actions ...Action) *Notification { acts := make([]*Action, len(actions)) for idx := range actions { a := actions[idx] @@ -145,8 +226,10 @@ func notify(nType Type, id, msg string, actions ...Action) *Notification { return Notify(&Notification{ EventID: id, Type: nType, + Title: title, Message: msg, AvailableActions: acts, + ShowOnSystem: showOnSystem, }) } @@ -185,15 +268,9 @@ func (n *Notification) save(pushUpdate bool) { n.lock.Lock() defer n.lock.Unlock() - // Move Title to Message, as that is the required field. - if n.Message == "" { - n.Message = n.Title - n.Title = "" - } - // Check if required data is present. - if n.Message == "" { - log.Warning("notifications: ignoring notification without Message") + if n.Title == "" && n.Message == "" { + log.Warning("notifications: ignoring notification without Title or Message") return } @@ -293,9 +370,8 @@ func (n *Notification) Update(expires int64) { } // Delete (prematurely) cancels and deletes a notification. -func (n *Notification) Delete() error { +func (n *Notification) Delete() { n.delete(true) - return nil } // delete deletes the notification from the internal storage. It locks the @@ -332,6 +408,8 @@ func (n *Notification) delete(pushUpdate bool) { if pushUpdate { dbController.PushUpdate(n) } + + n.resolveModuleFailure() } // Expired notifies the caller when the notification has expired. @@ -384,6 +462,7 @@ func (n *Notification) selectAndExecuteAction(id string) { if executed { n.State = Executed + n.resolveModuleFailure() } } From f18c0c5564e7c97f0b258468ab1e184164b7ae94 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 5 May 2021 00:13:50 +0200 Subject: [PATCH 03/16] Improve events and expose them via the runtime database --- config/main.go | 2 +- modules/events.go | 67 +++++++++++++++++++++++--- runtime/{module_api.go => module.go} | 6 +++ runtime/modules_integration.go | 72 ++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 8 deletions(-) rename runtime/{module_api.go => module.go} (86%) create mode 100644 runtime/modules_integration.go diff --git a/config/main.go b/config/main.go index b064b95..f98510e 100644 --- a/config/main.go +++ b/config/main.go @@ -32,7 +32,7 @@ func SetDataRoot(root *utils.DirStructure) { func init() { module = modules.Register("config", prep, start, nil, "database") - module.RegisterEvent(configChangeEvent) + module.RegisterEvent(configChangeEvent, true) flag.BoolVar(&exportConfig, "export-config-options", false, "export configuration registry and exit") } diff --git a/modules/events.go b/modules/events.go index 54e70aa..fd1a9bf 100644 --- a/modules/events.go +++ b/modules/events.go @@ -6,8 +6,18 @@ import ( "fmt" "github.com/safing/portbase/log" + "github.com/tevino/abool" ) +type eventHooks struct { + // hooks holds all registed hooks for the event. + hooks []*eventHook + + // internal signifies that the event and it's data may not be exposed and may + // only be propagated internally. + internal bool +} + type eventHookFn func(context.Context, interface{}) error type eventHook struct { @@ -27,17 +37,26 @@ func (m *Module) processEventTrigger(event string, data interface{}) { m.eventHooksLock.RLock() defer m.eventHooksLock.RUnlock() - hooks, ok := m.eventHooks[event] + eventHooks, ok := m.eventHooks[event] if !ok { log.Warningf(`%s: tried to trigger non-existent event "%s"`, m.Name, event) return } - for _, hook := range hooks { + for _, hook := range eventHooks.hooks { if hook.hookingModule.OnlineSoon() { go m.runEventHook(hook, event, data) } } + + // Call subscription function, if set. + if eventSubscriptionFuncReady.IsSet() { + m.StartWorker("event subscription", func(context.Context) error { + // Only use data in worker that won't change anymore. + eventSubscriptionFunc(m.Name, event, eventHooks.internal, data) + return nil + }) + } } // InjectEvent triggers an event from a foreign module and executes all hook functions registered to that event. @@ -63,12 +82,21 @@ func (m *Module) InjectEvent(sourceEventName, targetModuleName, targetEventName return fmt.Errorf(`module "%s" has no event named "%s"`, targetModuleName, targetEventName) } - for _, hook := range targetHooks { + for _, hook := range targetHooks.hooks { if hook.hookingModule.OnlineSoon() { go m.runEventHook(hook, sourceEventName, data) } } + // Call subscription function, if set. + if eventSubscriptionFuncReady.IsSet() { + m.StartWorker("event subscription", func(context.Context) error { + // Only use data in worker that won't change anymore. + eventSubscriptionFunc(targetModule.Name, targetEventName, targetHooks.internal, data) + return nil + }) + } + return nil } @@ -111,13 +139,20 @@ func (m *Module) runEventHook(hook *eventHook, event string, data interface{}) { } // RegisterEvent registers a new event to allow for registering hooks. -func (m *Module) RegisterEvent(event string) { +// The expose argument controls whether these events and the attached data may +// be received by external components via APIs. If not exposed, the database +// record that carries the event and it's data will be marked as secret and as +// a crown jewel. Enforcement is left to the database layer. +func (m *Module) RegisterEvent(event string, expose bool) { m.eventHooksLock.Lock() defer m.eventHooksLock.Unlock() _, ok := m.eventHooks[event] if !ok { - m.eventHooks[event] = make([]*eventHook, 0, 1) + m.eventHooks[event] = &eventHooks{ + hooks: make([]*eventHook, 0, 1), + internal: !expose, + } } } @@ -138,16 +173,34 @@ func (m *Module) RegisterEventHook(module string, event string, description stri // get target event eventModule.eventHooksLock.Lock() defer eventModule.eventHooksLock.Unlock() - hooks, ok := eventModule.eventHooks[event] + eventHooks, ok := eventModule.eventHooks[event] if !ok { return fmt.Errorf(`event "%s/%s" does not exist`, eventModule.Name, event) } // add hook - eventModule.eventHooks[event] = append(hooks, &eventHook{ + eventHooks.hooks = append(eventHooks.hooks, &eventHook{ description: description, hookingModule: m, hookFn: fn, }) return nil } + +// Subscribe to all events + +var ( + eventSubscriptionFunc func(moduleName, eventName string, internal bool, data interface{}) + eventSubscriptionFuncEnabled = abool.NewBool(false) + eventSubscriptionFuncReady = abool.NewBool(false) +) + +// SetEventSubscriptionFunc +func SetEventSubscriptionFunc(fn func(moduleName, eventName string, internal bool, data interface{})) bool { + if eventSubscriptionFuncEnabled.SetToIf(false, true) { + eventSubscriptionFunc = fn + eventSubscriptionFuncReady.Set() + return true + } + return false +} diff --git a/runtime/module_api.go b/runtime/module.go similarity index 86% rename from runtime/module_api.go rename to runtime/module.go index 2316c91..2f22f61 100644 --- a/runtime/module_api.go +++ b/runtime/module.go @@ -1,6 +1,8 @@ package runtime import ( + "fmt" + "github.com/safing/portbase/database" "github.com/safing/portbase/modules" ) @@ -30,6 +32,10 @@ func startModule() error { return err } + if err := startModulesIntegration(); err != nil { + return fmt.Errorf("failed to start modules integration: %w", err) + } + return nil } diff --git a/runtime/modules_integration.go b/runtime/modules_integration.go new file mode 100644 index 0000000..2a16c47 --- /dev/null +++ b/runtime/modules_integration.go @@ -0,0 +1,72 @@ +package runtime + +import ( + "fmt" + "sync" + + "github.com/safing/portbase/database" + "github.com/safing/portbase/database/record" + "github.com/safing/portbase/log" + "github.com/safing/portbase/modules" +) + +var ( + modulesIntegrationUpdatePusher func(...record.Record) +) + +func startModulesIntegration() (err error) { + modulesIntegrationUpdatePusher, err = Register("modules/", &ModulesIntegration{}) + if err != nil { + return err + } + + if !modules.SetEventSubscriptionFunc(pushModuleEvent) { + log.Warningf("runtime: failed to register the modules event subscription function") + } + + return nil +} + +type ModulesIntegration struct{} + +// Set is called when the value is set from outside. +// If the runtime value is considered read-only ErrReadOnly +// should be returned. It is guaranteed that the key of +// the record passed to Set is prefixed with the key used +// to register the value provider. +func (mi *ModulesIntegration) Set(record.Record) (record.Record, error) { + return nil, ErrReadOnly +} + +// Get should return one or more records that match keyOrPrefix. +// keyOrPrefix is guaranteed to be at least the prefix used to +// register the ValueProvider. +func (mi *ModulesIntegration) Get(keyOrPrefix string) ([]record.Record, error) { + return nil, database.ErrNotFound +} + +type eventData struct { //nolint:unused // This is a cascading false positive. + record.Base + sync.Mutex + Data interface{} +} + +func pushModuleEvent(moduleName, eventName string, internal bool, data interface{}) { //nolint:unused // This is a false positive, the function is provided to modules.SetEventSubscriptionFunc(). + // Create event record and set key. + eventRecord := &eventData{ + Data: data, + } + eventRecord.SetKey(fmt.Sprintf( + "runtime:modules/%s/event/%s", + moduleName, + eventName, + )) + eventRecord.UpdateMeta() + if internal { + eventRecord.Meta().MakeSecret() + eventRecord.Meta().MakeCrownJewel() + } + + // Push event to database subscriptions. + modulesIntegrationUpdatePusher(eventRecord) +} From 69330181db6355581d8e4e34672567f4d6a26446 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 5 May 2021 00:19:33 +0200 Subject: [PATCH 04/16] Fix test script and tests --- template/module.go | 2 +- test | 5 ++++- updater/resource_test.go | 8 ++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/template/module.go b/template/module.go index 72097ad..580b7fa 100644 --- a/template/module.go +++ b/template/module.go @@ -39,7 +39,7 @@ func init() { ) // register events that other modules can subscribe to - module.RegisterEvent(eventStateUpdate) + module.RegisterEvent(eventStateUpdate, true) } func prep() error { diff --git a/test b/test index d510075..ef569db 100755 --- a/test +++ b/test @@ -175,13 +175,16 @@ echo "running tests for ${platformInfo//$'\n'/ }:" # run vet/test on packages for package in $packages; do + package=${package#github.com/safing/portbase} + package=${package#/} + package=$PWD/$package echo "" echo $package if [[ $testonly -eq 0 ]]; then checkformat $package run golint -set_exit_status -min_confidence 1.0 $package run go vet $package - run golangci-lint run $GOPATH/src/$package + run golangci-lint run $package fi run go test -cover $fullTestFlags $package done diff --git a/updater/resource_test.go b/updater/resource_test.go index 2f734cc..038e4fe 100644 --- a/updater/resource_test.go +++ b/updater/resource_test.go @@ -36,14 +36,14 @@ func TestVersionSelection(t *testing.T) { registry.Beta = true registry.DevMode = true res.selectVersion() - if res.SelectedVersion.VersionNumber != "0" { - t.Errorf("selected version should be 0, not %s", res.SelectedVersion.VersionNumber) + if res.SelectedVersion.VersionNumber != "0.0.0" { + t.Errorf("selected version should be 0.0.0, not %s", res.SelectedVersion.VersionNumber) } registry.DevMode = false res.selectVersion() - if res.SelectedVersion.VersionNumber != "1.2.4b" { - t.Errorf("selected version should be 1.2.4b, not %s", res.SelectedVersion.VersionNumber) + if res.SelectedVersion.VersionNumber != "1.2.4-b" { + t.Errorf("selected version should be 1.2.4-b, not %s", res.SelectedVersion.VersionNumber) } registry.Beta = false From 4688a8d4901d36ca09670869905bc29526d7a75a Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 5 May 2021 13:01:17 +0200 Subject: [PATCH 05/16] Improve notification handling and documentation --- notifications/module-mirror.go | 30 +++++++++---- notifications/notification.go | 82 ++++++++++++++++++++-------------- 2 files changed, 69 insertions(+), 43 deletions(-) diff --git a/notifications/module-mirror.go b/notifications/module-mirror.go index 5d0fd2f..d26b288 100644 --- a/notifications/module-mirror.go +++ b/notifications/module-mirror.go @@ -8,8 +8,6 @@ import ( // AttachToModule attaches the notification to a module and changes to the // notification will be reflected on the module failure status. func (n *Notification) AttachToModule(m *modules.Module) { - log.Errorf("notifications: attaching %q", n.EventID) - if m == nil { log.Warningf("notifications: cannot remove attached module from notification %s", n.EventID) return @@ -46,8 +44,6 @@ func (n *Notification) AttachToModule(m *modules.Module) { // resolveModuleFailure removes the notification from the module failure status. func (n *Notification) resolveModuleFailure() { - log.Errorf("notifications: resolving %q", n.EventID) - if n.belongsTo != nil { // Resolve failure in attached module. n.belongsTo.Resolve(n.EventID) @@ -63,8 +59,6 @@ func init() { } func mirrorModuleStatus(moduleFailure uint8, id, title, msg string) { - log.Errorf("notifications: mirroring %d %q %q %q", moduleFailure, id, title, msg) - // Ignore "resolve all" requests. if id == "" { return @@ -84,12 +78,30 @@ func mirrorModuleStatus(moduleFailure uint8, id, title, msg string) { } // A notification for the given ID does not yet exists, create it. + n = &Notification{ + EventID: id, + Title: title, + Message: msg, + } + switch moduleFailure { case modules.FailureHint: - NotifyInfo(id, title, msg) + n.Type = Info case modules.FailureWarning: - NotifyWarn(id, title, msg) + n.Type = Warning + n.AvailableActions = []*Action{ + { + Text: "Get Help", + Type: ActionTypeOpenURL, + Payload: "https://safing.io/support/", + }, + } + + fallthrough case modules.FailureError: - NotifyError(id, title, msg) + n.Type = Error + n.ShowOnSystem = true } + + Notify(n) } diff --git a/notifications/notification.go b/notifications/notification.go index 6e6615c..68be10a 100644 --- a/notifications/notification.go +++ b/notifications/notification.go @@ -73,7 +73,8 @@ type Notification struct { // have any paramerized values replaced. Message string // ShowOnSystem specifies if the notification should be also shown on the - // operating system. + // operating system. Notifications shown on the operating system level are + // more focus-intrusive and should only be used for important notifications. ShowOnSystem bool // EventData contains an additional payload for the notification. This payload // may contain contextual data and may be used by a localization framework @@ -108,13 +109,19 @@ type Notification struct { // Action describes an action that can be taken for a notification. type Action struct { - ID string - Text string - Type ActionType + // ID specifies a unique ID for the action. If an action is selected, the ID + // is written to SelectedActionID and the notification is saved. + // If the action type is not ActionTypeNone, the ID may be empty, signifying + // that this action is merely additional and selecting it does dismiss the + // notification. + ID string + // Text on the button. + Text string + // Type specifies the action type. Implementing interfaces should only + // display action types they can handle. + Type ActionType + // Payload holds additional data for special action types. Payload interface{} - - // Dismisses specifies if the notification is dismissed when this action is selected. - Dismisses bool } // ActionType defines a specific type of action. @@ -184,43 +191,60 @@ func Delete(id string) { n, ok = nots[id] } -// NotifyInfo is a helper method for quickly showing a info -// notification. The notification is already shown. If id is -// an empty string a new UUIDv4 will be generated. +// NotifyInfo is a helper method for quickly showing an info notification. +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. // ShowOnSystem is disabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. func NotifyInfo(id, title, msg string, actions ...Action) *Notification { return notify(Info, id, title, msg, false, actions...) } -// NotifyWarn is a helper method for quickly showing a warning -// notification. The notification is already shown. If id is -// an empty string a new UUIDv4 will be generated. +// NotifyWarn is a helper method for quickly showing a warning notification +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. // ShowOnSystem is enabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. func NotifyWarn(id, title, msg string, actions ...Action) *Notification { return notify(Warning, id, title, msg, true, actions...) } -// NotifyError is a helper method for quickly showing an error -// notification. The notification is already shown. If id is -// an empty string a new UUIDv4 will be generated. +// NotifyError is a helper method for quickly showing an error notification. +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. // ShowOnSystem is enabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. func NotifyError(id, title, msg string, actions ...Action) *Notification { return notify(Error, id, title, msg, true, actions...) } -// NotifyPrompt is a helper method for quickly showing a prompt -// notification. The notification is already shown. If id is -// an empty string a new UUIDv4 will be generated. +// NotifyPrompt is a helper method for quickly showing a prompt notification. +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. // ShowOnSystem is disabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. func NotifyPrompt(id, title, msg string, actions ...Action) *Notification { return notify(Prompt, id, title, msg, false, actions...) } func notify(nType Type, id, title, msg string, showOnSystem bool, actions ...Action) *Notification { - acts := make([]*Action, len(actions)) - for idx := range actions { - a := actions[idx] - acts[idx] = &a + // Process actions. + var acts []*Action + if len(actions) == 0 { + // Create ack action if there are no defined actions. + acts = []*Action{ + { + ID: "ack", + Text: "OK", + }, + } + } else { + // Reference given actions for notification. + acts = make([]*Action, len(actions)) + for index := range actions { + a := actions[index] + acts[index] = &a + } } return Notify(&Notification{ @@ -228,8 +252,8 @@ func notify(nType Type, id, title, msg string, showOnSystem bool, actions ...Act Type: nType, Title: title, Message: msg, - AvailableActions: acts, ShowOnSystem: showOnSystem, + AvailableActions: acts, }) } @@ -290,16 +314,6 @@ func (n *Notification) save(pushUpdate bool) { n.GUID = utils.RandomUUID(n.EventID).String() } - // Make ack notification if there are no defined actions. - if len(n.AvailableActions) == 0 { - n.AvailableActions = []*Action{ - { - ID: "ack", - Text: "OK", - }, - } - } - // Make sure we always have a notification state assigned. if n.State == "" { n.State = Active From 215735175724a81eb6805a83eb73b4359a6ae0e2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 5 May 2021 17:45:33 +0200 Subject: [PATCH 06/16] Add internal tasks call, rename Prioritize to QueuePrioritized --- modules/tasks.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/tasks.go b/modules/tasks.go index 8c1bb93..1fbdec6 100644 --- a/modules/tasks.go +++ b/modules/tasks.go @@ -61,7 +61,10 @@ const ( defaultMaxDelay = 1 * time.Minute ) -// NewTask creates a new task with a descriptive name (non-unique), a optional deadline, and the task function to be executed. You must call one of Queue, Prioritize, StartASAP, Schedule or Repeat in order to have the Task executed. +// NewTask creates a new task with a descriptive name (non-unique), a optional +// deadline, and the task function to be executed. You must call one of Queue, +// QueuePrioritized, StartASAP, Schedule or Repeat in order to have the Task +// executed. func (m *Module) NewTask(name string, fn func(context.Context, *Task) error) *Task { if m == nil { log.Errorf(`modules: cannot create task "%s" with nil module`, name) @@ -75,6 +78,10 @@ func (m *Module) NewTask(name string, fn func(context.Context, *Task) error) *Ta m.Lock() defer m.Unlock() + return m.newTask(name, fn) +} + +func (m *Module) newTask(name string, fn func(context.Context, *Task) error) *Task { if m.Ctx == nil || !m.OnlineSoon() { log.Errorf(`modules: tasks should only be started when the module is online or starting`) return &Task{ @@ -144,8 +151,8 @@ func (t *Task) Queue() *Task { return t } -// Prioritize puts the task in the prioritized queue. -func (t *Task) Prioritize() *Task { +// QueuePrioritized queues the Task for execution in the prioritized queue. +func (t *Task) QueuePrioritized() *Task { t.lock.Lock() defer t.lock.Unlock() From 3f3786b8542b52efdf106afd41fc9d4acf44b52d Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 5 May 2021 17:46:09 +0200 Subject: [PATCH 07/16] Improve module failure status notify callback handling --- modules/status.go | 77 ++++++++++++++++++++-------------- notifications/module-mirror.go | 2 +- notifications/notification.go | 2 +- 3 files changed, 47 insertions(+), 34 deletions(-) diff --git a/modules/status.go b/modules/status.go index 3cc727a..3e9dd43 100644 --- a/modules/status.go +++ b/modules/status.go @@ -104,29 +104,34 @@ func (m *Module) Error(id, title, msg string) { } func (m *Module) setFailure(status uint8, id, title, msg string) { - // Send an update before we override a previous failure. - if failureUpdateNotifyFuncReady.IsSet() && m.failureID != "" { - updateFailureID := m.failureID - m.StartWorker("failure status updater", func(context.Context) error { - // Only use data in worker that won't change anymore. - failureUpdateNotifyFunc(FailureNone, updateFailureID, "", "") - return nil - }) - } + // Copy data for failure status update worker. + resolveFailureID := m.failureID + // Set new failure status. m.failureStatus = status m.failureID = id m.failureTitle = title m.failureMsg = msg - if failureUpdateNotifyFuncReady.IsSet() { - m.StartWorker("failure status updater", func(context.Context) error { - // Only use data in worker that won't change anymore. - failureUpdateNotifyFunc(status, id, title, msg) - return nil - }) - } + // Notify of module change. m.notifyOfChange() + + // Propagate failure status. + if failureUpdateNotifyFuncReady.IsSet() { + m.newTask("failure status updater", func(context.Context, *Task) error { + // Only use data in worker that won't change anymore. + + // Resolve previous failure state if available. + if resolveFailureID != "" { + failureUpdateNotifyFunc(FailureNone, resolveFailureID, "", "") + } + + // Notify of new failure state. + failureUpdateNotifyFunc(status, id, title, msg) + + return nil + }).QueuePrioritized() + } } // Resolve removes the failure state from the module if the given failureID matches the current failure ID. If the given failureID is an empty string, Resolve removes any failure state. @@ -134,24 +139,32 @@ func (m *Module) Resolve(failureID string) { m.Lock() defer m.Unlock() - if failureID == "" || failureID == m.failureID { - // Propagate resolving. - if failureUpdateNotifyFuncReady.IsSet() { - updateFailureID := m.failureID - m.StartWorker("failure status updater", func(context.Context) error { - // Only use data in worker that won't change anymore. - failureUpdateNotifyFunc(FailureNone, updateFailureID, "", "") - return nil - }) - } + // Check if resolving is necessary. + if failureID != "" && failureID != m.failureID { + // Return immediately if not resolving any (`""`) or if the failure ID + // does not match. + return + } - // Set failure status on module. - m.failureStatus = FailureNone - m.failureID = "" - m.failureTitle = "" - m.failureMsg = "" + // Copy data for failure status update worker. + resolveFailureID := m.failureID - m.notifyOfChange() + // Set failure status on module. + m.failureStatus = FailureNone + m.failureID = "" + m.failureTitle = "" + m.failureMsg = "" + + // Notify of module change. + m.notifyOfChange() + + // Propagate failure status. + if failureUpdateNotifyFuncReady.IsSet() { + m.newTask("failure status updater", func(context.Context, *Task) error { + // Only use data in worker that won't change anymore. + failureUpdateNotifyFunc(FailureNone, resolveFailureID, "", "") + return nil + }).QueuePrioritized() } } diff --git a/notifications/module-mirror.go b/notifications/module-mirror.go index d26b288..f633ca5 100644 --- a/notifications/module-mirror.go +++ b/notifications/module-mirror.go @@ -9,7 +9,7 @@ import ( // notification will be reflected on the module failure status. func (n *Notification) AttachToModule(m *modules.Module) { if m == nil { - log.Warningf("notifications: cannot remove attached module from notification %s", n.EventID) + log.Warningf("notifications: invalid usage: cannot attach %s to nil module", n.EventID) return } diff --git a/notifications/notification.go b/notifications/notification.go index 68be10a..0deef9d 100644 --- a/notifications/notification.go +++ b/notifications/notification.go @@ -112,7 +112,7 @@ type Action struct { // ID specifies a unique ID for the action. If an action is selected, the ID // is written to SelectedActionID and the notification is saved. // If the action type is not ActionTypeNone, the ID may be empty, signifying - // that this action is merely additional and selecting it does dismiss the + // that this action is merely additional and selecting it does not dismiss the // notification. ID string // Text on the button. From 0061572e1bb191dcb0ec01dc3e89673e165bd10b Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 11 May 2021 14:57:53 +0200 Subject: [PATCH 08/16] Add GetMeta to database storage interface --- database/controller.go | 24 ++++++++++- database/interface.go | 62 +++++++++++++++++++++++++-- database/query/README.md | 42 +++++++++--------- database/registry.go | 4 +- database/storage/badger/badger.go | 12 ++++++ database/storage/bbolt/bbolt.go | 12 ++++++ database/storage/fstree/fstree.go | 12 ++++++ database/storage/hashmap/map.go | 12 ++++++ database/storage/injectbase.go | 16 ++++--- database/storage/interface.go | 1 + database/storage/sinkhole/sinkhole.go | 5 +++ 11 files changed, 170 insertions(+), 32 deletions(-) diff --git a/database/controller.go b/database/controller.go index e015cfb..bf5de07 100644 --- a/database/controller.go +++ b/database/controller.go @@ -42,7 +42,7 @@ func (c *Controller) Injected() bool { return c.storage.Injected() } -// Get return the record with the given key. +// Get returns the record with the given key. func (c *Controller) Get(key string) (record.Record, error) { if shuttingDown.IsSet() { return nil, ErrShuttingDown @@ -76,6 +76,28 @@ func (c *Controller) Get(key string) (record.Record, error) { return r, nil } +// Get returns the metadata of the record with the given key. +func (c *Controller) GetMeta(key string) (*record.Meta, error) { + if shuttingDown.IsSet() { + return nil, ErrShuttingDown + } + + m, err := c.storage.GetMeta(key) + if err != nil { + // replace not found error + if err == storage.ErrNotFound { + return nil, ErrNotFound + } + return nil, err + } + + if !m.CheckValidity() { + return nil, ErrNotFound + } + + return m, nil +} + // Put saves a record in the database, executes any registered // pre-put hooks and finally send an update to all subscribers. // The record must be locked and secured from concurrent access diff --git a/database/interface.go b/database/interface.go index d0a6552..d3d18fb 100644 --- a/database/interface.go +++ b/database/interface.go @@ -201,6 +201,40 @@ func (i *Interface) getRecord(dbName string, dbKey string, mustBeWriteable bool) return r, db, nil } +func (i *Interface) getMeta(dbName string, dbKey string, mustBeWriteable bool) (m *record.Meta, db *Controller, err error) { + if dbName == "" { + dbName, dbKey = record.ParseKey(dbKey) + } + + db, err = getController(dbName) + if err != nil { + return nil, nil, err + } + + if mustBeWriteable && db.ReadOnly() { + return nil, db, ErrReadOnly + } + + r := i.checkCache(dbName + ":" + dbKey) + if r != nil { + if !i.options.hasAccessPermission(r) { + return nil, db, ErrPermissionDenied + } + return r.Meta(), db, nil + } + + m, err = db.GetMeta(dbKey) + if err != nil { + return nil, db, err + } + + if !m.CheckPermission(i.options.Local, i.options.Internal) { + return nil, db, ErrPermissionDenied + } + + return m, db, nil +} + // InsertValue inserts a value into a record. func (i *Interface) InsertValue(key string, attribute string, value interface{}) error { r, db, err := i.getRecord(getDBFromKey, key, true) @@ -236,7 +270,7 @@ func (i *Interface) Put(r record.Record) (err error) { // get record or only database var db *Controller if !i.options.HasAllPermissions() { - _, db, err = i.getRecord(r.DatabaseName(), r.DatabaseKey(), true) + _, db, err = i.getMeta(r.DatabaseName(), r.DatabaseKey(), true) if err != nil && err != ErrNotFound { return err } @@ -247,7 +281,7 @@ func (i *Interface) Put(r record.Record) (err error) { } } - // Check if database is read only before we add to the cache. + // Check if database is read only. if db.ReadOnly() { return ErrReadOnly } @@ -274,7 +308,7 @@ func (i *Interface) PutNew(r record.Record) (err error) { // get record or only database var db *Controller if !i.options.HasAllPermissions() { - _, db, err = i.getRecord(r.DatabaseName(), r.DatabaseKey(), true) + _, db, err = i.getMeta(r.DatabaseName(), r.DatabaseKey(), true) if err != nil && err != ErrNotFound { return err } @@ -285,6 +319,11 @@ func (i *Interface) PutNew(r record.Record) (err error) { } } + // Check if database is read only. + if db.ReadOnly() { + return ErrReadOnly + } + r.Lock() if r.Meta() != nil { r.Meta().Reset() @@ -328,6 +367,13 @@ func (i *Interface) PutMany(dbName string) (put func(record.Record) error) { } } + // Check if database is read only. + if db.ReadOnly() { + return func(r record.Record) error { + return ErrReadOnly + } + } + // start database access dbBatch, errs := db.PutMany() finished := abool.New() @@ -462,6 +508,11 @@ func (i *Interface) Delete(key string) error { return err } + // Check if database is read only. + if db.ReadOnly() { + return ErrReadOnly + } + i.options.Apply(r) r.Meta().Delete() return db.Put(r) @@ -495,6 +546,11 @@ func (i *Interface) Purge(ctx context.Context, q *query.Query) (int, error) { return 0, err } + // Check if database is read only before we add to the cache. + if db.ReadOnly() { + return 0, ErrReadOnly + } + return db.Purge(ctx, q, i.options.Local, i.options.Internal) } diff --git a/database/query/README.md b/database/query/README.md index 9311417..455eb95 100644 --- a/database/query/README.md +++ b/database/query/README.md @@ -21,28 +21,28 @@ Please note that some feeders may have other special characters. It is advised t ## Operators -| Name | Textual | Req. Type | Internal Type | Compared with | -|---|---|---|---| -| Equals | `==` | int | int64 | `==` | -| GreaterThan | `>` | int | int64 | `>` | -| GreaterThanOrEqual | `>=` | int | int64 | `>=` | -| LessThan | `<` | int | int64 | `<` | -| LessThanOrEqual | `<=` | int | int64 | `<=` | -| FloatEquals | `f==` | float | float64 | `==` | -| FloatGreaterThan | `f>` | float | float64 | `>` | -| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` | -| FloatLessThan | `f<` | float | float64 | `<` | -| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` | -| SameAs | `sameas`, `s==` | string | string | `==` | -| Contains | `contains`, `co` | string | string | `strings.Contains()` | -| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` | -| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` | -| In | `in` | string | string | for loop with `==` | -| Matches | `matches`, `re` | string | int64 | `regexp.Regexp.Matches()` | -| Is | `is` | bool* | bool | `==` | -| Exists | `exists`, `ex` | any | n/a | n/a | +| Name | Textual | Req. Type | Internal Type | Compared with | +|-------------------------|--------------------|-----------|---------------|---------------------------| +| Equals | `==` | int | int64 | `==` | +| GreaterThan | `>` | int | int64 | `>` | +| GreaterThanOrEqual | `>=` | int | int64 | `>=` | +| LessThan | `<` | int | int64 | `<` | +| LessThanOrEqual | `<=` | int | int64 | `<=` | +| FloatEquals | `f==` | float | float64 | `==` | +| FloatGreaterThan | `f>` | float | float64 | `>` | +| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` | +| FloatLessThan | `f<` | float | float64 | `<` | +| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` | +| SameAs | `sameas`, `s==` | string | string | `==` | +| Contains | `contains`, `co` | string | string | `strings.Contains()` | +| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` | +| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` | +| In | `in` | string | string | for loop with `==` | +| Matches | `matches`, `re` | string | string | `regexp.Regexp.Matches()` | +| Is | `is` | bool* | bool | `==` | +| Exists | `exists`, `ex` | any | n/a | n/a | -\*accepts strings: 1, t, T, TRUE, true, True, 0, f, F, FALSE +\*accepts strings: 1, t, T, true, True, TRUE, 0, f, F, false, False, FALSE ## Escaping diff --git a/database/registry.go b/database/registry.go index 81d1de4..2e27b6c 100644 --- a/database/registry.go +++ b/database/registry.go @@ -25,7 +25,7 @@ var ( registry = make(map[string]*Database) registryLock sync.Mutex - nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{4,}$") + nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{3,}$") ) // Register registers a new database. @@ -56,7 +56,7 @@ func Register(new *Database) (*Database, error) { } else { // register new database if !nameConstraint.MatchString(new.Name) { - return nil, errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 4 characters long") + return nil, errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 3 characters long") } now := time.Now().Round(time.Second) diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go index dd0487c..2501f9b 100644 --- a/database/storage/badger/badger.go +++ b/database/storage/badger/badger.go @@ -82,6 +82,18 @@ func (b *Badger) Get(key string) (record.Record, error) { return m, nil } +// GetMeta returns the metadata of a database record. +func (b *Badger) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := b.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (b *Badger) Put(r record.Record) (record.Record, error) { data, err := r.MarshalRecord(r) diff --git a/database/storage/bbolt/bbolt.go b/database/storage/bbolt/bbolt.go index 214605c..48cd0d0 100644 --- a/database/storage/bbolt/bbolt.go +++ b/database/storage/bbolt/bbolt.go @@ -96,6 +96,18 @@ func (b *BBolt) Get(key string) (record.Record, error) { return r, nil } +// GetMeta returns the metadata of a database record. +func (b *BBolt) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := b.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (b *BBolt) Put(r record.Record) (record.Record, error) { data, err := r.MarshalRecord(r) diff --git a/database/storage/fstree/fstree.go b/database/storage/fstree/fstree.go index a96f914..0cf72d3 100644 --- a/database/storage/fstree/fstree.go +++ b/database/storage/fstree/fstree.go @@ -104,6 +104,18 @@ func (fst *FSTree) Get(key string) (record.Record, error) { return r, nil } +// GetMeta returns the metadata of a database record. +func (fst *FSTree) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := fst.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (fst *FSTree) Put(r record.Record) (record.Record, error) { dstPath, err := fst.buildFilePath(r.DatabaseKey(), true) diff --git a/database/storage/hashmap/map.go b/database/storage/hashmap/map.go index 87c6272..eb25f24 100644 --- a/database/storage/hashmap/map.go +++ b/database/storage/hashmap/map.go @@ -44,6 +44,18 @@ func (hm *HashMap) Get(key string) (record.Record, error) { return r, nil } +// GetMeta returns the metadata of a database record. +func (hm *HashMap) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := hm.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (hm *HashMap) Put(r record.Record) (record.Record, error) { hm.dbLock.Lock() diff --git a/database/storage/injectbase.go b/database/storage/injectbase.go index 467c1ba..b91257f 100644 --- a/database/storage/injectbase.go +++ b/database/storage/injectbase.go @@ -11,7 +11,8 @@ import ( ) var ( - errNotImplemented = errors.New("not implemented") + // ErrNotImplemented is returned when a function is not implemented by a storage. + ErrNotImplemented = errors.New("not implemented") ) // InjectBase is a dummy base structure to reduce boilerplate code for injected storage interfaces. @@ -22,22 +23,27 @@ var _ Interface = &InjectBase{} // Get returns a database record. func (i *InjectBase) Get(key string) (record.Record, error) { - return nil, errNotImplemented + return nil, ErrNotImplemented +} + +// Get returns a database record. +func (i *InjectBase) GetMeta(key string) (*record.Meta, error) { + return nil, ErrNotImplemented } // Put stores a record in the database. func (i *InjectBase) Put(m record.Record) (record.Record, error) { - return nil, errNotImplemented + return nil, ErrNotImplemented } // Delete deletes a record from the database. func (i *InjectBase) Delete(key string) error { - return errNotImplemented + return ErrNotImplemented } // Query returns a an iterator for the supplied query. func (i *InjectBase) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) { - return nil, errNotImplemented + return nil, ErrNotImplemented } // ReadOnly returns whether the database is read only. diff --git a/database/storage/interface.go b/database/storage/interface.go index b1cfcb1..a7d111e 100644 --- a/database/storage/interface.go +++ b/database/storage/interface.go @@ -13,6 +13,7 @@ import ( type Interface interface { // Primary Interface Get(key string) (record.Record, error) + GetMeta(key string) (*record.Meta, error) Put(m record.Record) (record.Record, error) Delete(key string) error Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) diff --git a/database/storage/sinkhole/sinkhole.go b/database/storage/sinkhole/sinkhole.go index d033638..af6b082 100644 --- a/database/storage/sinkhole/sinkhole.go +++ b/database/storage/sinkhole/sinkhole.go @@ -44,6 +44,11 @@ func (s *Sinkhole) Get(key string) (record.Record, error) { return nil, storage.ErrNotFound } +// GetMeta returns the metadata of a database record. +func (s *Sinkhole) GetMeta(key string) (*record.Meta, error) { + return nil, storage.ErrNotFound +} + // Put stores a record in the database. func (s *Sinkhole) Put(r record.Record) (record.Record, error) { return r, nil From 745a27d92d9d5014ca05974144dc14f280a9a4df Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 11 May 2021 14:58:28 +0200 Subject: [PATCH 09/16] Fix required API methods --- api/endpoints_meta.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/endpoints_meta.go b/api/endpoints_meta.go index 0cf4189..9077307 100644 --- a/api/endpoints_meta.go +++ b/api/endpoints_meta.go @@ -51,6 +51,7 @@ func registerMetaEndpoints() error { if err := RegisterEndpoint(Endpoint{ Path: "auth/reset", Read: PermitAnyone, + Write: PermitAnyone, HandlerFunc: authReset, Name: "Reset Authenticated Session", Description: "Resets authentication status internally and in the browser.", From 8c1c522475cb5cbca8bbdc9bc14974566a0afb25 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 11 May 2021 14:59:00 +0200 Subject: [PATCH 10/16] Use workers for database api handlers --- api/database.go | 86 +++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/api/database.go b/api/database.go index d893f10..f23523f 100644 --- a/api/database.go +++ b/api/database.go @@ -2,6 +2,7 @@ package api import ( "bytes" + "context" "errors" "fmt" "net/http" @@ -38,7 +39,8 @@ const ( ) var ( - dbAPISeperatorBytes = []byte(dbAPISeperator) + dbAPISeperatorBytes = []byte(dbAPISeperator) + dbCompatibilityPermission = PermitAdmin ) func init() { @@ -46,8 +48,8 @@ func init() { startDatabaseAPI, // Default to admin read/write permissions until the database gets support // for api permissions. - PermitAdmin, - PermitAdmin, + dbCompatibilityPermission, + dbCompatibilityPermission, )) } @@ -96,13 +98,13 @@ func startDatabaseAPI(w http.ResponseWriter, r *http.Request) { db: database.NewInterface(nil), } - go new.handler() - go new.writer() + module.StartWorker("database api handler", new.handler) + module.StartWorker("database api writer", new.writer) log.Tracer(r.Context()).Infof("api request: init websocket %s %s", r.RemoteAddr, r.RequestURI) } -func (api *DatabaseAPI) handler() { +func (api *DatabaseAPI) handler(context.Context) error { // 123|get| // 123|ok|| @@ -146,19 +148,7 @@ func (api *DatabaseAPI) handler() { _, msg, err := api.conn.ReadMessage() if err != nil { - if !api.shuttingDown.IsSet() { - api.shutdown() - if websocket.IsCloseError(err, - websocket.CloseNormalClosure, - websocket.CloseGoingAway, - websocket.CloseAbnormalClosure, - ) { - log.Infof("api: websocket connection to %s closed", api.conn.RemoteAddr()) - } else { - log.Warningf("api: websocket read error from %s: %s", api.conn.RemoteAddr(), err) - } - } - return + return api.shutdown(err) } parts := bytes.SplitN(msg, []byte("|"), 3) @@ -218,45 +208,56 @@ func (api *DatabaseAPI) handler() { } } -func (api *DatabaseAPI) writer() { +func (api *DatabaseAPI) writer(ctx context.Context) error { var data []byte var err error for { - data = nil - select { // prioritize direct writes case data = <-api.sendQueue: if len(data) == 0 { - api.shutdown() - return + return api.shutdown(nil) } + case <-ctx.Done(): + return api.shutdown(nil) case <-api.shutdownSignal: - return + return api.shutdown(nil) } // log.Tracef("api: sending %s", string(*msg)) err = api.conn.WriteMessage(websocket.BinaryMessage, data) if err != nil { - if !api.shuttingDown.IsSet() { - api.shutdown() - if websocket.IsCloseError(err, - websocket.CloseNormalClosure, - websocket.CloseGoingAway, - websocket.CloseAbnormalClosure, - ) { - log.Infof("api: websocket connection to %s closed", api.conn.RemoteAddr()) - } else { - log.Warningf("api: websocket write error to %s: %s", api.conn.RemoteAddr(), err) - } - } - return + return api.shutdown(err) } - } } +func (api *DatabaseAPI) shutdown(err error) error { + // Check if we are the first to shut down. + if !api.shuttingDown.SetToIf(false, true) { + return nil + } + + // Check the given error. + if err != nil { + if websocket.IsCloseError(err, + websocket.CloseNormalClosure, + websocket.CloseGoingAway, + websocket.CloseAbnormalClosure, + ) { + log.Infof("api: websocket connection to %s closed", api.conn.RemoteAddr()) + } else { + log.Warningf("api: websocket connection error with %s: %s", api.conn.RemoteAddr(), err) + } + } + + // Trigger shutdown. + close(api.shutdownSignal) + api.conn.Close() + return nil +} + func (api *DatabaseAPI) send(opID []byte, msgType string, msgOrKey string, data []byte) { c := container.New(opID) c.Append(dbAPISeperatorBytes) @@ -622,13 +623,6 @@ func (api *DatabaseAPI) handleDelete(opID []byte, key string) { api.send(opID, dbMsgTypeSuccess, emptyString, nil) } -func (api *DatabaseAPI) shutdown() { - if api.shuttingDown.SetToIf(false, true) { - close(api.shutdownSignal) - api.conn.Close() - } -} - // marsharlRecords locks and marshals the given record, additionally adding // metadata and returning it as json. func marshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) { From 8814d279bdfdd63975b4b153ac7036e482b70926 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 11 May 2021 14:59:33 +0200 Subject: [PATCH 11/16] Add api bridge for database --- api/api_bridge.go | 178 ++++++++++++++++++++++++++++++++++++++++++ api/authentication.go | 8 ++ api/endpoints.go | 10 +-- api/main.go | 2 +- 4 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 api/api_bridge.go diff --git a/api/api_bridge.go b/api/api_bridge.go new file mode 100644 index 0000000..0e147a1 --- /dev/null +++ b/api/api_bridge.go @@ -0,0 +1,178 @@ +package api + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "path" + "strings" + "sync" + + "github.com/safing/portbase/database" + "github.com/safing/portbase/database/record" + "github.com/safing/portbase/database/storage" + "github.com/safing/portbase/log" +) + +const ( + endpointBridgeRemoteAddress = "websocket-bridge" + apiDatabaseName = "api" +) + +func registerEndpointBridgeDB() error { + if _, err := database.Register(&database.Database{ + Name: apiDatabaseName, + Description: "API Bridge", + StorageType: "injected", + }); err != nil { + return err + } + + _, err := database.InjectDatabase("api", &endpointBridgeStorage{}) + return err +} + +type endpointBridgeStorage struct { + storage.InjectBase +} + +type EndpointBridgeRequest struct { + record.Base + sync.Mutex + + Method string + Path string + Query map[string]string + Data []byte + MimeType string +} + +type EndpointBridgeResponse struct { + record.Base + sync.Mutex + + MimeType string + Body string +} + +// Get returns a database record. +func (ebs *endpointBridgeStorage) Get(key string) (record.Record, error) { + log.Errorf("api bridge: getting %s", key) + + if key == "" { + return nil, database.ErrNotFound + } + + return callAPI(&EndpointBridgeRequest{ + Method: http.MethodGet, + Path: key, + }) +} + +// Get returns the metadata of a database record. +func (ebs *endpointBridgeStorage) GetMeta(key string) (*record.Meta, error) { + // This interface is an API, always return a fresh copy. + m := &record.Meta{} + m.Update() + return m, nil +} + +// Put stores a record in the database. +func (ebs *endpointBridgeStorage) Put(r record.Record) (record.Record, error) { + log.Errorf("api bridge: putting %s", r.Key()) + + if r.DatabaseKey() == "" { + return nil, database.ErrNotFound + } + + // Prepare data. + var ebr *EndpointBridgeRequest + if r.IsWrapped() { + // Only allocate a new struct, if we need it. + ebr = &EndpointBridgeRequest{} + err := record.Unwrap(r, ebr) + if err != nil { + return nil, err + } + } else { + var ok bool + ebr, ok = r.(*EndpointBridgeRequest) + if !ok { + return nil, fmt.Errorf("record not of type *EndpointBridgeRequest, but %T", r) + } + } + log.Errorf("api bridge: putting %+v", ebr) + + // Override path with key to mitigate sneaky stuff. + ebr.Path = r.DatabaseKey() + return callAPI(ebr) +} + +// ReadOnly returns whether the database is read only. +func (ebs *endpointBridgeStorage) ReadOnly() bool { + return false +} + +func callAPI(ebr *EndpointBridgeRequest) (record.Record, error) { + // Add API prefix to path. + requestURL := path.Join(apiV1Path, ebr.Path) + // Check if path is correct. (Defense in depth) + if !strings.HasPrefix(requestURL, apiV1Path) { + return nil, fmt.Errorf("bridged request for %q violates scope", ebr.Path) + } + + // Apply default Method. + if ebr.Method == "" { + if len(ebr.Data) > 0 { + ebr.Method = http.MethodPost + } else { + ebr.Method = http.MethodGet + } + } + + // Build URL. + u, err := url.ParseRequestURI(requestURL) + if err != nil { + return nil, fmt.Errorf("failed to build bridged request url: %w", err) + } + // Build query values. + if ebr.Query != nil && len(ebr.Query) > 0 { + query := url.Values{} + for k, v := range ebr.Query { + query.Set(k, v) + } + u.RawQuery = query.Encode() + } + log.Errorf("api bridge: calling %s", u.String()) + + // Create request and response objects. + r := httptest.NewRequest(ebr.Method, u.String(), bytes.NewBuffer(ebr.Data)) + r.RemoteAddr = endpointBridgeRemoteAddress + if ebr.MimeType != "" { + r.Header.Set("Content-Type", ebr.MimeType) + } + w := httptest.NewRecorder() + // Let the API handle the request. + server.Handler.ServeHTTP(w, r) + switch w.Code { + case 200: + // Everything okay, continue. + case 500: + // A Go error was returned internally. + // We can safely return this as an error. + return nil, fmt.Errorf("bridged api call failed: %s", w.Body.String()) + default: + return nil, fmt.Errorf("bridged api call returned unexpected error code %d", w.Code) + } + + response := &EndpointBridgeResponse{ + MimeType: w.Result().Header.Get("Content-Type"), + Body: w.Body.String(), + } + response.SetKey(apiDatabaseName + ":" + ebr.Path) + response.UpdateMeta() + + return response, nil +} diff --git a/api/authentication.go b/api/authentication.go index bbac2c9..3e8be8e 100644 --- a/api/authentication.go +++ b/api/authentication.go @@ -250,6 +250,14 @@ func checkAuth(w http.ResponseWriter, r *http.Request, authRequired bool) (token }, false } + // Database Bridge Access. + if r.RemoteAddr == endpointBridgeRemoteAddress { + return &AuthToken{ + Read: dbCompatibilityPermission, + Write: dbCompatibilityPermission, + }, false + } + // Check for valid API key. token = checkAPIKey(r) if token != nil { diff --git a/api/endpoints.go b/api/endpoints.go index bdb068d..885477c 100644 --- a/api/endpoints.go +++ b/api/endpoints.go @@ -260,7 +260,7 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) return default: - http.Error(w, "Unsupported method for the actions API.", http.StatusMethodNotAllowed) + http.Error(w, "unsupported method for the actions API", http.StatusMethodNotAllowed) return } @@ -298,13 +298,13 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return default: - http.Error(w, "Internal server error: Missing handler.", http.StatusInternalServerError) + http.Error(w, "missing handler", http.StatusInternalServerError) return } // Check for handler error. if err != nil { - http.Error(w, "Internal server error: "+err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -321,14 +321,14 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func readBody(w http.ResponseWriter, r *http.Request) (inputData []byte, ok bool) { // Check for too long content in order to prevent death. if r.ContentLength > 20000000 { // 20MB - http.Error(w, "Too much input data.", http.StatusRequestEntityTooLarge) + http.Error(w, "too much input data", http.StatusRequestEntityTooLarge) return nil, false } // Read and close body. inputData, err := ioutil.ReadAll(r.Body) if err != nil { - http.Error(w, "Failed to read body: "+err.Error(), http.StatusInternalServerError) + http.Error(w, "failed to read body"+err.Error(), http.StatusInternalServerError) return nil, false } return inputData, true diff --git a/api/main.go b/api/main.go index 17e2a6d..1c86044 100644 --- a/api/main.go +++ b/api/main.go @@ -67,7 +67,7 @@ func start() error { module.NewTask("clean api sessions", cleanSessions).Repeat(5 * time.Minute) } - return nil + return registerEndpointBridgeDB() } func stop() error { From 400f4c12edd708e021578f84694e49117e3de5a9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 12 May 2021 11:24:30 +0200 Subject: [PATCH 12/16] Use gorilla/mux for endpoint paths --- api/api_bridge.go | 7 ------ api/endpoints.go | 63 +++++++++++++++++++++++++++++++++-------------- api/main.go | 4 +++ 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/api/api_bridge.go b/api/api_bridge.go index 0e147a1..905c201 100644 --- a/api/api_bridge.go +++ b/api/api_bridge.go @@ -13,7 +13,6 @@ import ( "github.com/safing/portbase/database" "github.com/safing/portbase/database/record" "github.com/safing/portbase/database/storage" - "github.com/safing/portbase/log" ) const ( @@ -59,8 +58,6 @@ type EndpointBridgeResponse struct { // Get returns a database record. func (ebs *endpointBridgeStorage) Get(key string) (record.Record, error) { - log.Errorf("api bridge: getting %s", key) - if key == "" { return nil, database.ErrNotFound } @@ -81,8 +78,6 @@ func (ebs *endpointBridgeStorage) GetMeta(key string) (*record.Meta, error) { // Put stores a record in the database. func (ebs *endpointBridgeStorage) Put(r record.Record) (record.Record, error) { - log.Errorf("api bridge: putting %s", r.Key()) - if r.DatabaseKey() == "" { return nil, database.ErrNotFound } @@ -103,7 +98,6 @@ func (ebs *endpointBridgeStorage) Put(r record.Record) (record.Record, error) { return nil, fmt.Errorf("record not of type *EndpointBridgeRequest, but %T", r) } } - log.Errorf("api bridge: putting %+v", ebr) // Override path with key to mitigate sneaky stuff. ebr.Path = r.DatabaseKey() @@ -145,7 +139,6 @@ func callAPI(ebr *EndpointBridgeRequest) (record.Record, error) { } u.RawQuery = query.Encode() } - log.Errorf("api bridge: calling %s", u.String()) // Create request and response objects. r := httptest.NewRequest(ebr.Method, u.String(), bytes.NewBuffer(ebr.Data)) diff --git a/api/endpoints.go b/api/endpoints.go index 885477c..3241a70 100644 --- a/api/endpoints.go +++ b/api/endpoints.go @@ -11,6 +11,8 @@ import ( "strings" "sync" + "github.com/gorilla/mux" + "github.com/safing/portbase/database/record" "github.com/safing/portbase/log" ) @@ -84,6 +86,7 @@ func init() { var ( endpoints = make(map[string]*Endpoint) + endpointsMux = mux.NewRouter() endpointsLock sync.RWMutex // ErrInvalidEndpoint is returned when an invalid endpoint is registered. @@ -106,16 +109,28 @@ func getAPIContext(r *http.Request) (apiEndpoint *Endpoint, apiRequest *Request) return apiEndpoint, apiRequest } - // If not, get the action from the registry. - endpointPath, ok := apiRequest.URLVars["endpointPath"] - if !ok { - return nil, apiRequest - } - endpointsLock.RLock() defer endpointsLock.RUnlock() - apiEndpoint, ok = endpoints[endpointPath] + // Get handler for request. + // Gorilla does not support handling this on our own very well. + // See github.com/gorilla/mux.ServeHTTP for reference. + var match mux.RouteMatch + var handler http.Handler + if endpointsMux.Match(r, &match) { + handler = match.Handler + apiRequest.Route = match.Route + // Add/Override variables instead of replacing. + for k, v := range match.Vars { + apiRequest.URLVars[k] = v + } + } else { + return nil, apiRequest + } + + log.Errorf("handler: %+v", handler) + apiEndpoint, ok = handler.(*Endpoint) + log.Errorf("apiEndpoint: %+v", apiEndpoint) if ok { // Cache for next operation. apiRequest.HandlerCache = apiEndpoint @@ -139,6 +154,7 @@ func RegisterEndpoint(e Endpoint) error { } endpoints[e.Path] = &e + endpointsMux.Handle(apiV1Path+e.Path, &e) return nil } @@ -243,6 +259,17 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + apiEndpoint.ServeHTTP(w, r) +} + +// ServeHTTP handles the http request. +func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) { + _, apiRequest := getAPIContext(r) + if apiRequest == nil { + http.NotFound(w, r) + return + } + switch r.Method { case http.MethodHead: w.WriteHeader(http.StatusOK) @@ -269,32 +296,32 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var err error switch { - case apiEndpoint.ActionFunc != nil: + case e.ActionFunc != nil: var msg string - msg, err = apiEndpoint.ActionFunc(apiRequest) + msg, err = e.ActionFunc(apiRequest) if err == nil { responseData = []byte(msg) } - case apiEndpoint.DataFunc != nil: - responseData, err = apiEndpoint.DataFunc(apiRequest) + case e.DataFunc != nil: + responseData, err = e.DataFunc(apiRequest) - case apiEndpoint.StructFunc != nil: + case e.StructFunc != nil: var v interface{} - v, err = apiEndpoint.StructFunc(apiRequest) + v, err = e.StructFunc(apiRequest) if err == nil && v != nil { responseData, err = json.Marshal(v) } - case apiEndpoint.RecordFunc != nil: + case e.RecordFunc != nil: var rec record.Record - rec, err = apiEndpoint.RecordFunc(apiRequest) + rec, err = e.RecordFunc(apiRequest) if err == nil && r != nil { responseData, err = marshalRecord(rec, false) } - case apiEndpoint.HandlerFunc != nil: - apiEndpoint.HandlerFunc(w, r) + case e.HandlerFunc != nil: + e.HandlerFunc(w, r) return default: @@ -309,7 +336,7 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // Write response. - w.Header().Set("Content-Type", apiEndpoint.MimeType+"; charset=utf-8") + w.Header().Set("Content-Type", e.MimeType+"; charset=utf-8") w.Header().Set("Content-Length", strconv.Itoa(len(responseData))) w.WriteHeader(http.StatusOK) _, err = w.Write(responseData) diff --git a/api/main.go b/api/main.go index 1c86044..68172c8 100644 --- a/api/main.go +++ b/api/main.go @@ -50,6 +50,10 @@ func prep() error { return err } + if err := registerModulesEndpoints(); err != nil { + return err + } + return registerMetaEndpoints() } From a41ea62d2d46dedd3208ca77c2b175345927d40c Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 12 May 2021 11:24:44 +0200 Subject: [PATCH 13/16] Add api for injecting events --- api/endpoints_modules.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 api/endpoints_modules.go diff --git a/api/endpoints_modules.go b/api/endpoints_modules.go new file mode 100644 index 0000000..da124b9 --- /dev/null +++ b/api/endpoints_modules.go @@ -0,0 +1,36 @@ +package api + +import ( + "errors" + "fmt" +) + +func registerModulesEndpoints() error { + if err := RegisterEndpoint(Endpoint{ + Path: "modules/{moduleName:.+}/trigger/{eventName:.+}", + Write: PermitSelf, + ActionFunc: triggerEvent, + Name: "Export Configuration Options", + Description: "Returns a list of all registered configuration options and their metadata. This does not include the current active or default settings.", + }); err != nil { + return err + } + + return nil +} + +func triggerEvent(ar *Request) (msg string, err error) { + // Get parameters. + moduleName := ar.URLVars["moduleName"] + eventName := ar.URLVars["eventName"] + if moduleName == "" || eventName == "" { + return "", errors.New("invalid parameters") + } + + // Inject event. + if err := module.InjectEvent("api event injection", moduleName, eventName, nil); err != nil { + return "", fmt.Errorf("failed to inject event: %w", err) + } + + return "event successfully injected", nil +} From 5dc39685da94db9a288876c23e222a283f5b2a75 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 12 May 2021 15:36:35 +0200 Subject: [PATCH 14/16] Move GetMeta storage API to separate interface --- database/controller.go | 28 +++++++++++++++++++++------- database/storage/interface.go | 6 +++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/database/controller.go b/database/controller.go index bf5de07..cb89cf5 100644 --- a/database/controller.go +++ b/database/controller.go @@ -55,7 +55,7 @@ func (c *Controller) Get(key string) (record.Record, error) { r, err := c.storage.Get(key) if err != nil { // replace not found error - if err == storage.ErrNotFound { + if errors.Is(err, storage.ErrNotFound) { return nil, ErrNotFound } return nil, err @@ -82,13 +82,27 @@ func (c *Controller) GetMeta(key string) (*record.Meta, error) { return nil, ErrShuttingDown } - m, err := c.storage.GetMeta(key) - if err != nil { - // replace not found error - if err == storage.ErrNotFound { - return nil, ErrNotFound + var m *record.Meta + var err error + if metaDB, ok := c.storage.(storage.MetaHandler); ok { + m, err = metaDB.GetMeta(key) + if err != nil { + // replace not found error + if errors.Is(err, storage.ErrNotFound) { + return nil, ErrNotFound + } + return nil, err } - return nil, err + } else { + r, err := c.storage.Get(key) + if err != nil { + // replace not found error + if errors.Is(err, storage.ErrNotFound) { + return nil, ErrNotFound + } + return nil, err + } + m = r.Meta() } if !m.CheckValidity() { diff --git a/database/storage/interface.go b/database/storage/interface.go index a7d111e..7546485 100644 --- a/database/storage/interface.go +++ b/database/storage/interface.go @@ -13,7 +13,6 @@ import ( type Interface interface { // Primary Interface Get(key string) (record.Record, error) - GetMeta(key string) (*record.Meta, error) Put(m record.Record) (record.Record, error) Delete(key string) error Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) @@ -27,6 +26,11 @@ type Interface interface { MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time, shadowDelete bool) error } +// Maintainer defines the database storage API for backends that support optimized fetching of only the metadata. +type MetaHandler interface { + GetMeta(key string) (*record.Meta, error) +} + // Maintainer defines the database storage API for backends that require regular maintenance. type Maintainer interface { Maintain(ctx context.Context) error From c26b18edb4a54d6f1a2d4d669f8269628434680f Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 12 May 2021 15:36:51 +0200 Subject: [PATCH 15/16] Fix module failure mirror --- modules/status.go | 27 ++++++++++++++++++++++++--- notifications/module-mirror.go | 20 ++++++++++---------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/modules/status.go b/modules/status.go index 3e9dd43..fb7a93d 100644 --- a/modules/status.go +++ b/modules/status.go @@ -79,7 +79,14 @@ func (m *Module) FailureStatus() (failureStatus uint8, failureID, failureMsg str return m.failureStatus, m.failureID, m.failureMsg } -// Hint sets failure status to hint. This is a somewhat special failure status, as the module is believed to be working correctly, but there is an important module specific information to convey. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. +// Hint sets failure status to hint. This is a somewhat special failure status, +// as the module is believed to be working correctly, but there is an important +// module specific information to convey. The supplied failureID is for +// improved automatic handling within connected systems, the failureMsg is for +// humans. +// The given ID must be unique for the given title and message. A call to +// Hint(), Warning() or Error() with the same ID as the existing one will be +// ignored. func (m *Module) Hint(id, title, msg string) { m.Lock() defer m.Unlock() @@ -87,7 +94,12 @@ func (m *Module) Hint(id, title, msg string) { m.setFailure(FailureHint, id, title, msg) } -// Warning sets failure status to warning. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. +// Warning sets failure status to warning. The supplied failureID is for +// improved automatic handling within connected systems, the failureMsg is for +// humans. +// The given ID must be unique for the given title and message. A call to +// Hint(), Warning() or Error() with the same ID as the existing one will be +// ignored. func (m *Module) Warning(id, title, msg string) { m.Lock() defer m.Unlock() @@ -95,7 +107,11 @@ func (m *Module) Warning(id, title, msg string) { m.setFailure(FailureWarning, id, title, msg) } -// Error sets failure status to error. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. +// Error sets failure status to error. The supplied failureID is for improved +// automatic handling within connected systems, the failureMsg is for humans. +// The given ID must be unique for the given title and message. A call to +// Hint(), Warning() or Error() with the same ID as the existing one will be +// ignored. func (m *Module) Error(id, title, msg string) { m.Lock() defer m.Unlock() @@ -104,6 +120,11 @@ func (m *Module) Error(id, title, msg string) { } func (m *Module) setFailure(status uint8, id, title, msg string) { + // Ignore calls with the same ID. + if id == m.failureID { + return + } + // Copy data for failure status update worker. resolveFailureID := m.failureID diff --git a/notifications/module-mirror.go b/notifications/module-mirror.go index f633ca5..24f0741 100644 --- a/notifications/module-mirror.go +++ b/notifications/module-mirror.go @@ -82,22 +82,22 @@ func mirrorModuleStatus(moduleFailure uint8, id, title, msg string) { EventID: id, Title: title, Message: msg, - } - - switch moduleFailure { - case modules.FailureHint: - n.Type = Info - case modules.FailureWarning: - n.Type = Warning - n.AvailableActions = []*Action{ + AvailableActions: []*Action{ { Text: "Get Help", Type: ActionTypeOpenURL, Payload: "https://safing.io/support/", }, - } + }, + } - fallthrough + switch moduleFailure { + case modules.FailureHint: + n.Type = Info + n.AvailableActions = nil + case modules.FailureWarning: + n.Type = Warning + n.ShowOnSystem = true case modules.FailureError: n.Type = Error n.ShowOnSystem = true From 8c1f2fcbe8af7cc26d5e44ee9366fb4589b512ad Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 12 May 2021 15:37:03 +0200 Subject: [PATCH 16/16] Remove debug msgs --- api/endpoints.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/endpoints.go b/api/endpoints.go index 3241a70..c02b9d0 100644 --- a/api/endpoints.go +++ b/api/endpoints.go @@ -128,9 +128,7 @@ func getAPIContext(r *http.Request) (apiEndpoint *Endpoint, apiRequest *Request) return nil, apiRequest } - log.Errorf("handler: %+v", handler) apiEndpoint, ok = handler.(*Endpoint) - log.Errorf("apiEndpoint: %+v", apiEndpoint) if ok { // Cache for next operation. apiRequest.HandlerCache = apiEndpoint