diff --git a/api/api_bridge.go b/api/api_bridge.go new file mode 100644 index 0000000..905c201 --- /dev/null +++ b/api/api_bridge.go @@ -0,0 +1,171 @@ +package api + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "path" + "strings" + "sync" + + "github.com/safing/portbase/database" + "github.com/safing/portbase/database/record" + "github.com/safing/portbase/database/storage" +) + +const ( + endpointBridgeRemoteAddress = "websocket-bridge" + apiDatabaseName = "api" +) + +func registerEndpointBridgeDB() error { + if _, err := database.Register(&database.Database{ + Name: apiDatabaseName, + Description: "API Bridge", + StorageType: "injected", + }); err != nil { + return err + } + + _, err := database.InjectDatabase("api", &endpointBridgeStorage{}) + return err +} + +type endpointBridgeStorage struct { + storage.InjectBase +} + +type EndpointBridgeRequest struct { + record.Base + sync.Mutex + + Method string + Path string + Query map[string]string + Data []byte + MimeType string +} + +type EndpointBridgeResponse struct { + record.Base + sync.Mutex + + MimeType string + Body string +} + +// Get returns a database record. +func (ebs *endpointBridgeStorage) Get(key string) (record.Record, error) { + if key == "" { + return nil, database.ErrNotFound + } + + return callAPI(&EndpointBridgeRequest{ + Method: http.MethodGet, + Path: key, + }) +} + +// Get returns the metadata of a database record. +func (ebs *endpointBridgeStorage) GetMeta(key string) (*record.Meta, error) { + // This interface is an API, always return a fresh copy. + m := &record.Meta{} + m.Update() + return m, nil +} + +// Put stores a record in the database. +func (ebs *endpointBridgeStorage) Put(r record.Record) (record.Record, error) { + if r.DatabaseKey() == "" { + return nil, database.ErrNotFound + } + + // Prepare data. + var ebr *EndpointBridgeRequest + if r.IsWrapped() { + // Only allocate a new struct, if we need it. + ebr = &EndpointBridgeRequest{} + err := record.Unwrap(r, ebr) + if err != nil { + return nil, err + } + } else { + var ok bool + ebr, ok = r.(*EndpointBridgeRequest) + if !ok { + return nil, fmt.Errorf("record not of type *EndpointBridgeRequest, but %T", r) + } + } + + // Override path with key to mitigate sneaky stuff. + ebr.Path = r.DatabaseKey() + return callAPI(ebr) +} + +// ReadOnly returns whether the database is read only. +func (ebs *endpointBridgeStorage) ReadOnly() bool { + return false +} + +func callAPI(ebr *EndpointBridgeRequest) (record.Record, error) { + // Add API prefix to path. + requestURL := path.Join(apiV1Path, ebr.Path) + // Check if path is correct. (Defense in depth) + if !strings.HasPrefix(requestURL, apiV1Path) { + return nil, fmt.Errorf("bridged request for %q violates scope", ebr.Path) + } + + // Apply default Method. + if ebr.Method == "" { + if len(ebr.Data) > 0 { + ebr.Method = http.MethodPost + } else { + ebr.Method = http.MethodGet + } + } + + // Build URL. + u, err := url.ParseRequestURI(requestURL) + if err != nil { + return nil, fmt.Errorf("failed to build bridged request url: %w", err) + } + // Build query values. + if ebr.Query != nil && len(ebr.Query) > 0 { + query := url.Values{} + for k, v := range ebr.Query { + query.Set(k, v) + } + u.RawQuery = query.Encode() + } + + // Create request and response objects. + r := httptest.NewRequest(ebr.Method, u.String(), bytes.NewBuffer(ebr.Data)) + r.RemoteAddr = endpointBridgeRemoteAddress + if ebr.MimeType != "" { + r.Header.Set("Content-Type", ebr.MimeType) + } + w := httptest.NewRecorder() + // Let the API handle the request. + server.Handler.ServeHTTP(w, r) + switch w.Code { + case 200: + // Everything okay, continue. + case 500: + // A Go error was returned internally. + // We can safely return this as an error. + return nil, fmt.Errorf("bridged api call failed: %s", w.Body.String()) + default: + return nil, fmt.Errorf("bridged api call returned unexpected error code %d", w.Code) + } + + response := &EndpointBridgeResponse{ + MimeType: w.Result().Header.Get("Content-Type"), + Body: w.Body.String(), + } + response.SetKey(apiDatabaseName + ":" + ebr.Path) + response.UpdateMeta() + + return response, nil +} diff --git a/api/authentication.go b/api/authentication.go index bbac2c9..3e8be8e 100644 --- a/api/authentication.go +++ b/api/authentication.go @@ -250,6 +250,14 @@ func checkAuth(w http.ResponseWriter, r *http.Request, authRequired bool) (token }, false } + // Database Bridge Access. + if r.RemoteAddr == endpointBridgeRemoteAddress { + return &AuthToken{ + Read: dbCompatibilityPermission, + Write: dbCompatibilityPermission, + }, false + } + // Check for valid API key. token = checkAPIKey(r) if token != nil { diff --git a/api/config.go b/api/config.go index ebcc515..354ea2a 100644 --- a/api/config.go +++ b/api/config.go @@ -4,7 +4,6 @@ import ( "flag" "github.com/safing/portbase/config" - "github.com/safing/portbase/log" ) // Config Keys. @@ -24,13 +23,12 @@ var ( ) func init() { - flag.StringVar(&listenAddressFlag, "api-address", "", "override api listen address") -} - -func logFlagOverrides() { - if listenAddressFlag != "" { - log.Warning("api: api/listenAddress default config is being overridden by -api-address flag") - } + flag.StringVar( + &listenAddressFlag, + "api-address", + "", + "set api listen address; configuration is stronger", + ) } func getDefaultListenAddress() string { diff --git a/api/database.go b/api/database.go index d893f10..f23523f 100644 --- a/api/database.go +++ b/api/database.go @@ -2,6 +2,7 @@ package api import ( "bytes" + "context" "errors" "fmt" "net/http" @@ -38,7 +39,8 @@ const ( ) var ( - dbAPISeperatorBytes = []byte(dbAPISeperator) + dbAPISeperatorBytes = []byte(dbAPISeperator) + dbCompatibilityPermission = PermitAdmin ) func init() { @@ -46,8 +48,8 @@ func init() { startDatabaseAPI, // Default to admin read/write permissions until the database gets support // for api permissions. - PermitAdmin, - PermitAdmin, + dbCompatibilityPermission, + dbCompatibilityPermission, )) } @@ -96,13 +98,13 @@ func startDatabaseAPI(w http.ResponseWriter, r *http.Request) { db: database.NewInterface(nil), } - go new.handler() - go new.writer() + module.StartWorker("database api handler", new.handler) + module.StartWorker("database api writer", new.writer) log.Tracer(r.Context()).Infof("api request: init websocket %s %s", r.RemoteAddr, r.RequestURI) } -func (api *DatabaseAPI) handler() { +func (api *DatabaseAPI) handler(context.Context) error { // 123|get| // 123|ok|| @@ -146,19 +148,7 @@ func (api *DatabaseAPI) handler() { _, msg, err := api.conn.ReadMessage() if err != nil { - if !api.shuttingDown.IsSet() { - api.shutdown() - if websocket.IsCloseError(err, - websocket.CloseNormalClosure, - websocket.CloseGoingAway, - websocket.CloseAbnormalClosure, - ) { - log.Infof("api: websocket connection to %s closed", api.conn.RemoteAddr()) - } else { - log.Warningf("api: websocket read error from %s: %s", api.conn.RemoteAddr(), err) - } - } - return + return api.shutdown(err) } parts := bytes.SplitN(msg, []byte("|"), 3) @@ -218,45 +208,56 @@ func (api *DatabaseAPI) handler() { } } -func (api *DatabaseAPI) writer() { +func (api *DatabaseAPI) writer(ctx context.Context) error { var data []byte var err error for { - data = nil - select { // prioritize direct writes case data = <-api.sendQueue: if len(data) == 0 { - api.shutdown() - return + return api.shutdown(nil) } + case <-ctx.Done(): + return api.shutdown(nil) case <-api.shutdownSignal: - return + return api.shutdown(nil) } // log.Tracef("api: sending %s", string(*msg)) err = api.conn.WriteMessage(websocket.BinaryMessage, data) if err != nil { - if !api.shuttingDown.IsSet() { - api.shutdown() - if websocket.IsCloseError(err, - websocket.CloseNormalClosure, - websocket.CloseGoingAway, - websocket.CloseAbnormalClosure, - ) { - log.Infof("api: websocket connection to %s closed", api.conn.RemoteAddr()) - } else { - log.Warningf("api: websocket write error to %s: %s", api.conn.RemoteAddr(), err) - } - } - return + return api.shutdown(err) } - } } +func (api *DatabaseAPI) shutdown(err error) error { + // Check if we are the first to shut down. + if !api.shuttingDown.SetToIf(false, true) { + return nil + } + + // Check the given error. + if err != nil { + if websocket.IsCloseError(err, + websocket.CloseNormalClosure, + websocket.CloseGoingAway, + websocket.CloseAbnormalClosure, + ) { + log.Infof("api: websocket connection to %s closed", api.conn.RemoteAddr()) + } else { + log.Warningf("api: websocket connection error with %s: %s", api.conn.RemoteAddr(), err) + } + } + + // Trigger shutdown. + close(api.shutdownSignal) + api.conn.Close() + return nil +} + func (api *DatabaseAPI) send(opID []byte, msgType string, msgOrKey string, data []byte) { c := container.New(opID) c.Append(dbAPISeperatorBytes) @@ -622,13 +623,6 @@ func (api *DatabaseAPI) handleDelete(opID []byte, key string) { api.send(opID, dbMsgTypeSuccess, emptyString, nil) } -func (api *DatabaseAPI) shutdown() { - if api.shuttingDown.SetToIf(false, true) { - close(api.shutdownSignal) - api.conn.Close() - } -} - // marsharlRecords locks and marshals the given record, additionally adding // metadata and returning it as json. func marshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) { diff --git a/api/endpoints.go b/api/endpoints.go index bdb068d..c02b9d0 100644 --- a/api/endpoints.go +++ b/api/endpoints.go @@ -11,6 +11,8 @@ import ( "strings" "sync" + "github.com/gorilla/mux" + "github.com/safing/portbase/database/record" "github.com/safing/portbase/log" ) @@ -84,6 +86,7 @@ func init() { var ( endpoints = make(map[string]*Endpoint) + endpointsMux = mux.NewRouter() endpointsLock sync.RWMutex // ErrInvalidEndpoint is returned when an invalid endpoint is registered. @@ -106,16 +109,26 @@ func getAPIContext(r *http.Request) (apiEndpoint *Endpoint, apiRequest *Request) return apiEndpoint, apiRequest } - // If not, get the action from the registry. - endpointPath, ok := apiRequest.URLVars["endpointPath"] - if !ok { - return nil, apiRequest - } - endpointsLock.RLock() defer endpointsLock.RUnlock() - apiEndpoint, ok = endpoints[endpointPath] + // Get handler for request. + // Gorilla does not support handling this on our own very well. + // See github.com/gorilla/mux.ServeHTTP for reference. + var match mux.RouteMatch + var handler http.Handler + if endpointsMux.Match(r, &match) { + handler = match.Handler + apiRequest.Route = match.Route + // Add/Override variables instead of replacing. + for k, v := range match.Vars { + apiRequest.URLVars[k] = v + } + } else { + return nil, apiRequest + } + + apiEndpoint, ok = handler.(*Endpoint) if ok { // Cache for next operation. apiRequest.HandlerCache = apiEndpoint @@ -139,6 +152,7 @@ func RegisterEndpoint(e Endpoint) error { } endpoints[e.Path] = &e + endpointsMux.Handle(apiV1Path+e.Path, &e) return nil } @@ -243,6 +257,17 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + apiEndpoint.ServeHTTP(w, r) +} + +// ServeHTTP handles the http request. +func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) { + _, apiRequest := getAPIContext(r) + if apiRequest == nil { + http.NotFound(w, r) + return + } + switch r.Method { case http.MethodHead: w.WriteHeader(http.StatusOK) @@ -260,7 +285,7 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) return default: - http.Error(w, "Unsupported method for the actions API.", http.StatusMethodNotAllowed) + http.Error(w, "unsupported method for the actions API", http.StatusMethodNotAllowed) return } @@ -269,47 +294,47 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var err error switch { - case apiEndpoint.ActionFunc != nil: + case e.ActionFunc != nil: var msg string - msg, err = apiEndpoint.ActionFunc(apiRequest) + msg, err = e.ActionFunc(apiRequest) if err == nil { responseData = []byte(msg) } - case apiEndpoint.DataFunc != nil: - responseData, err = apiEndpoint.DataFunc(apiRequest) + case e.DataFunc != nil: + responseData, err = e.DataFunc(apiRequest) - case apiEndpoint.StructFunc != nil: + case e.StructFunc != nil: var v interface{} - v, err = apiEndpoint.StructFunc(apiRequest) + v, err = e.StructFunc(apiRequest) if err == nil && v != nil { responseData, err = json.Marshal(v) } - case apiEndpoint.RecordFunc != nil: + case e.RecordFunc != nil: var rec record.Record - rec, err = apiEndpoint.RecordFunc(apiRequest) + rec, err = e.RecordFunc(apiRequest) if err == nil && r != nil { responseData, err = marshalRecord(rec, false) } - case apiEndpoint.HandlerFunc != nil: - apiEndpoint.HandlerFunc(w, r) + case e.HandlerFunc != nil: + e.HandlerFunc(w, r) return default: - http.Error(w, "Internal server error: Missing handler.", http.StatusInternalServerError) + http.Error(w, "missing handler", http.StatusInternalServerError) return } // Check for handler error. if err != nil { - http.Error(w, "Internal server error: "+err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusInternalServerError) return } // Write response. - w.Header().Set("Content-Type", apiEndpoint.MimeType+"; charset=utf-8") + w.Header().Set("Content-Type", e.MimeType+"; charset=utf-8") w.Header().Set("Content-Length", strconv.Itoa(len(responseData))) w.WriteHeader(http.StatusOK) _, err = w.Write(responseData) @@ -321,14 +346,14 @@ func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func readBody(w http.ResponseWriter, r *http.Request) (inputData []byte, ok bool) { // Check for too long content in order to prevent death. if r.ContentLength > 20000000 { // 20MB - http.Error(w, "Too much input data.", http.StatusRequestEntityTooLarge) + http.Error(w, "too much input data", http.StatusRequestEntityTooLarge) return nil, false } // Read and close body. inputData, err := ioutil.ReadAll(r.Body) if err != nil { - http.Error(w, "Failed to read body: "+err.Error(), http.StatusInternalServerError) + http.Error(w, "failed to read body"+err.Error(), http.StatusInternalServerError) return nil, false } return inputData, true diff --git a/api/endpoints_meta.go b/api/endpoints_meta.go index 0cf4189..9077307 100644 --- a/api/endpoints_meta.go +++ b/api/endpoints_meta.go @@ -51,6 +51,7 @@ func registerMetaEndpoints() error { if err := RegisterEndpoint(Endpoint{ Path: "auth/reset", Read: PermitAnyone, + Write: PermitAnyone, HandlerFunc: authReset, Name: "Reset Authenticated Session", Description: "Resets authentication status internally and in the browser.", diff --git a/api/endpoints_modules.go b/api/endpoints_modules.go new file mode 100644 index 0000000..da124b9 --- /dev/null +++ b/api/endpoints_modules.go @@ -0,0 +1,36 @@ +package api + +import ( + "errors" + "fmt" +) + +func registerModulesEndpoints() error { + if err := RegisterEndpoint(Endpoint{ + Path: "modules/{moduleName:.+}/trigger/{eventName:.+}", + Write: PermitSelf, + ActionFunc: triggerEvent, + Name: "Export Configuration Options", + Description: "Returns a list of all registered configuration options and their metadata. This does not include the current active or default settings.", + }); err != nil { + return err + } + + return nil +} + +func triggerEvent(ar *Request) (msg string, err error) { + // Get parameters. + moduleName := ar.URLVars["moduleName"] + eventName := ar.URLVars["eventName"] + if moduleName == "" || eventName == "" { + return "", errors.New("invalid parameters") + } + + // Inject event. + if err := module.InjectEvent("api event injection", moduleName, eventName, nil); err != nil { + return "", fmt.Errorf("failed to inject event: %w", err) + } + + return "event successfully injected", nil +} diff --git a/api/main.go b/api/main.go index e47b618..68172c8 100644 --- a/api/main.go +++ b/api/main.go @@ -50,11 +50,14 @@ func prep() error { return err } + if err := registerModulesEndpoints(); err != nil { + return err + } + return registerMetaEndpoints() } func start() error { - logFlagOverrides() go Serve() _ = updateAPIKeys(module.Ctx, nil) @@ -68,7 +71,7 @@ func start() error { module.NewTask("clean api sessions", cleanSessions).Repeat(5 * time.Minute) } - return nil + return registerEndpointBridgeDB() } func stop() error { diff --git a/config/basic_config.go b/config/basic_config.go index ce4a716..6975231 100644 --- a/config/basic_config.go +++ b/config/basic_config.go @@ -18,7 +18,7 @@ var ( ) func init() { - flag.BoolVar(&defaultDevMode, "devmode", false, "enable development mode") + flag.BoolVar(&defaultDevMode, "devmode", false, "enable development mode; configuration is stronger") } func registerBasicOptions() error { diff --git a/config/main.go b/config/main.go index b064b95..f98510e 100644 --- a/config/main.go +++ b/config/main.go @@ -32,7 +32,7 @@ func SetDataRoot(root *utils.DirStructure) { func init() { module = modules.Register("config", prep, start, nil, "database") - module.RegisterEvent(configChangeEvent) + module.RegisterEvent(configChangeEvent, true) flag.BoolVar(&exportConfig, "export-config-options", false, "export configuration registry and exit") } diff --git a/database/controller.go b/database/controller.go index e015cfb..cb89cf5 100644 --- a/database/controller.go +++ b/database/controller.go @@ -42,7 +42,7 @@ func (c *Controller) Injected() bool { return c.storage.Injected() } -// Get return the record with the given key. +// Get returns the record with the given key. func (c *Controller) Get(key string) (record.Record, error) { if shuttingDown.IsSet() { return nil, ErrShuttingDown @@ -55,7 +55,7 @@ func (c *Controller) Get(key string) (record.Record, error) { r, err := c.storage.Get(key) if err != nil { // replace not found error - if err == storage.ErrNotFound { + if errors.Is(err, storage.ErrNotFound) { return nil, ErrNotFound } return nil, err @@ -76,6 +76,42 @@ func (c *Controller) Get(key string) (record.Record, error) { return r, nil } +// Get returns the metadata of the record with the given key. +func (c *Controller) GetMeta(key string) (*record.Meta, error) { + if shuttingDown.IsSet() { + return nil, ErrShuttingDown + } + + var m *record.Meta + var err error + if metaDB, ok := c.storage.(storage.MetaHandler); ok { + m, err = metaDB.GetMeta(key) + if err != nil { + // replace not found error + if errors.Is(err, storage.ErrNotFound) { + return nil, ErrNotFound + } + return nil, err + } + } else { + r, err := c.storage.Get(key) + if err != nil { + // replace not found error + if errors.Is(err, storage.ErrNotFound) { + return nil, ErrNotFound + } + return nil, err + } + m = r.Meta() + } + + if !m.CheckValidity() { + return nil, ErrNotFound + } + + return m, nil +} + // Put saves a record in the database, executes any registered // pre-put hooks and finally send an update to all subscribers. // The record must be locked and secured from concurrent access diff --git a/database/interface.go b/database/interface.go index d0a6552..d3d18fb 100644 --- a/database/interface.go +++ b/database/interface.go @@ -201,6 +201,40 @@ func (i *Interface) getRecord(dbName string, dbKey string, mustBeWriteable bool) return r, db, nil } +func (i *Interface) getMeta(dbName string, dbKey string, mustBeWriteable bool) (m *record.Meta, db *Controller, err error) { + if dbName == "" { + dbName, dbKey = record.ParseKey(dbKey) + } + + db, err = getController(dbName) + if err != nil { + return nil, nil, err + } + + if mustBeWriteable && db.ReadOnly() { + return nil, db, ErrReadOnly + } + + r := i.checkCache(dbName + ":" + dbKey) + if r != nil { + if !i.options.hasAccessPermission(r) { + return nil, db, ErrPermissionDenied + } + return r.Meta(), db, nil + } + + m, err = db.GetMeta(dbKey) + if err != nil { + return nil, db, err + } + + if !m.CheckPermission(i.options.Local, i.options.Internal) { + return nil, db, ErrPermissionDenied + } + + return m, db, nil +} + // InsertValue inserts a value into a record. func (i *Interface) InsertValue(key string, attribute string, value interface{}) error { r, db, err := i.getRecord(getDBFromKey, key, true) @@ -236,7 +270,7 @@ func (i *Interface) Put(r record.Record) (err error) { // get record or only database var db *Controller if !i.options.HasAllPermissions() { - _, db, err = i.getRecord(r.DatabaseName(), r.DatabaseKey(), true) + _, db, err = i.getMeta(r.DatabaseName(), r.DatabaseKey(), true) if err != nil && err != ErrNotFound { return err } @@ -247,7 +281,7 @@ func (i *Interface) Put(r record.Record) (err error) { } } - // Check if database is read only before we add to the cache. + // Check if database is read only. if db.ReadOnly() { return ErrReadOnly } @@ -274,7 +308,7 @@ func (i *Interface) PutNew(r record.Record) (err error) { // get record or only database var db *Controller if !i.options.HasAllPermissions() { - _, db, err = i.getRecord(r.DatabaseName(), r.DatabaseKey(), true) + _, db, err = i.getMeta(r.DatabaseName(), r.DatabaseKey(), true) if err != nil && err != ErrNotFound { return err } @@ -285,6 +319,11 @@ func (i *Interface) PutNew(r record.Record) (err error) { } } + // Check if database is read only. + if db.ReadOnly() { + return ErrReadOnly + } + r.Lock() if r.Meta() != nil { r.Meta().Reset() @@ -328,6 +367,13 @@ func (i *Interface) PutMany(dbName string) (put func(record.Record) error) { } } + // Check if database is read only. + if db.ReadOnly() { + return func(r record.Record) error { + return ErrReadOnly + } + } + // start database access dbBatch, errs := db.PutMany() finished := abool.New() @@ -462,6 +508,11 @@ func (i *Interface) Delete(key string) error { return err } + // Check if database is read only. + if db.ReadOnly() { + return ErrReadOnly + } + i.options.Apply(r) r.Meta().Delete() return db.Put(r) @@ -495,6 +546,11 @@ func (i *Interface) Purge(ctx context.Context, q *query.Query) (int, error) { return 0, err } + // Check if database is read only before we add to the cache. + if db.ReadOnly() { + return 0, ErrReadOnly + } + return db.Purge(ctx, q, i.options.Local, i.options.Internal) } diff --git a/database/query/README.md b/database/query/README.md index 9311417..455eb95 100644 --- a/database/query/README.md +++ b/database/query/README.md @@ -21,28 +21,28 @@ Please note that some feeders may have other special characters. It is advised t ## Operators -| Name | Textual | Req. Type | Internal Type | Compared with | -|---|---|---|---| -| Equals | `==` | int | int64 | `==` | -| GreaterThan | `>` | int | int64 | `>` | -| GreaterThanOrEqual | `>=` | int | int64 | `>=` | -| LessThan | `<` | int | int64 | `<` | -| LessThanOrEqual | `<=` | int | int64 | `<=` | -| FloatEquals | `f==` | float | float64 | `==` | -| FloatGreaterThan | `f>` | float | float64 | `>` | -| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` | -| FloatLessThan | `f<` | float | float64 | `<` | -| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` | -| SameAs | `sameas`, `s==` | string | string | `==` | -| Contains | `contains`, `co` | string | string | `strings.Contains()` | -| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` | -| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` | -| In | `in` | string | string | for loop with `==` | -| Matches | `matches`, `re` | string | int64 | `regexp.Regexp.Matches()` | -| Is | `is` | bool* | bool | `==` | -| Exists | `exists`, `ex` | any | n/a | n/a | +| Name | Textual | Req. Type | Internal Type | Compared with | +|-------------------------|--------------------|-----------|---------------|---------------------------| +| Equals | `==` | int | int64 | `==` | +| GreaterThan | `>` | int | int64 | `>` | +| GreaterThanOrEqual | `>=` | int | int64 | `>=` | +| LessThan | `<` | int | int64 | `<` | +| LessThanOrEqual | `<=` | int | int64 | `<=` | +| FloatEquals | `f==` | float | float64 | `==` | +| FloatGreaterThan | `f>` | float | float64 | `>` | +| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` | +| FloatLessThan | `f<` | float | float64 | `<` | +| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` | +| SameAs | `sameas`, `s==` | string | string | `==` | +| Contains | `contains`, `co` | string | string | `strings.Contains()` | +| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` | +| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` | +| In | `in` | string | string | for loop with `==` | +| Matches | `matches`, `re` | string | string | `regexp.Regexp.Matches()` | +| Is | `is` | bool* | bool | `==` | +| Exists | `exists`, `ex` | any | n/a | n/a | -\*accepts strings: 1, t, T, TRUE, true, True, 0, f, F, FALSE +\*accepts strings: 1, t, T, true, True, TRUE, 0, f, F, false, False, FALSE ## Escaping diff --git a/database/registry.go b/database/registry.go index 81d1de4..2e27b6c 100644 --- a/database/registry.go +++ b/database/registry.go @@ -25,7 +25,7 @@ var ( registry = make(map[string]*Database) registryLock sync.Mutex - nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{4,}$") + nameConstraint = regexp.MustCompile("^[A-Za-z0-9_-]{3,}$") ) // Register registers a new database. @@ -56,7 +56,7 @@ func Register(new *Database) (*Database, error) { } else { // register new database if !nameConstraint.MatchString(new.Name) { - return nil, errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 4 characters long") + return nil, errors.New("database name must only contain alphanumeric and `_-` characters and must be at least 3 characters long") } now := time.Now().Round(time.Second) diff --git a/database/storage/badger/badger.go b/database/storage/badger/badger.go index dd0487c..2501f9b 100644 --- a/database/storage/badger/badger.go +++ b/database/storage/badger/badger.go @@ -82,6 +82,18 @@ func (b *Badger) Get(key string) (record.Record, error) { return m, nil } +// GetMeta returns the metadata of a database record. +func (b *Badger) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := b.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (b *Badger) Put(r record.Record) (record.Record, error) { data, err := r.MarshalRecord(r) diff --git a/database/storage/bbolt/bbolt.go b/database/storage/bbolt/bbolt.go index 214605c..48cd0d0 100644 --- a/database/storage/bbolt/bbolt.go +++ b/database/storage/bbolt/bbolt.go @@ -96,6 +96,18 @@ func (b *BBolt) Get(key string) (record.Record, error) { return r, nil } +// GetMeta returns the metadata of a database record. +func (b *BBolt) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := b.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (b *BBolt) Put(r record.Record) (record.Record, error) { data, err := r.MarshalRecord(r) diff --git a/database/storage/fstree/fstree.go b/database/storage/fstree/fstree.go index a96f914..0cf72d3 100644 --- a/database/storage/fstree/fstree.go +++ b/database/storage/fstree/fstree.go @@ -104,6 +104,18 @@ func (fst *FSTree) Get(key string) (record.Record, error) { return r, nil } +// GetMeta returns the metadata of a database record. +func (fst *FSTree) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := fst.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (fst *FSTree) Put(r record.Record) (record.Record, error) { dstPath, err := fst.buildFilePath(r.DatabaseKey(), true) diff --git a/database/storage/hashmap/map.go b/database/storage/hashmap/map.go index 87c6272..eb25f24 100644 --- a/database/storage/hashmap/map.go +++ b/database/storage/hashmap/map.go @@ -44,6 +44,18 @@ func (hm *HashMap) Get(key string) (record.Record, error) { return r, nil } +// GetMeta returns the metadata of a database record. +func (hm *HashMap) GetMeta(key string) (*record.Meta, error) { + // TODO: Replace with more performant variant. + + r, err := hm.Get(key) + if err != nil { + return nil, err + } + + return r.Meta(), nil +} + // Put stores a record in the database. func (hm *HashMap) Put(r record.Record) (record.Record, error) { hm.dbLock.Lock() diff --git a/database/storage/injectbase.go b/database/storage/injectbase.go index 467c1ba..b91257f 100644 --- a/database/storage/injectbase.go +++ b/database/storage/injectbase.go @@ -11,7 +11,8 @@ import ( ) var ( - errNotImplemented = errors.New("not implemented") + // ErrNotImplemented is returned when a function is not implemented by a storage. + ErrNotImplemented = errors.New("not implemented") ) // InjectBase is a dummy base structure to reduce boilerplate code for injected storage interfaces. @@ -22,22 +23,27 @@ var _ Interface = &InjectBase{} // Get returns a database record. func (i *InjectBase) Get(key string) (record.Record, error) { - return nil, errNotImplemented + return nil, ErrNotImplemented +} + +// Get returns a database record. +func (i *InjectBase) GetMeta(key string) (*record.Meta, error) { + return nil, ErrNotImplemented } // Put stores a record in the database. func (i *InjectBase) Put(m record.Record) (record.Record, error) { - return nil, errNotImplemented + return nil, ErrNotImplemented } // Delete deletes a record from the database. func (i *InjectBase) Delete(key string) error { - return errNotImplemented + return ErrNotImplemented } // Query returns a an iterator for the supplied query. func (i *InjectBase) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) { - return nil, errNotImplemented + return nil, ErrNotImplemented } // ReadOnly returns whether the database is read only. diff --git a/database/storage/interface.go b/database/storage/interface.go index b1cfcb1..7546485 100644 --- a/database/storage/interface.go +++ b/database/storage/interface.go @@ -26,6 +26,11 @@ type Interface interface { MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time, shadowDelete bool) error } +// Maintainer defines the database storage API for backends that support optimized fetching of only the metadata. +type MetaHandler interface { + GetMeta(key string) (*record.Meta, error) +} + // Maintainer defines the database storage API for backends that require regular maintenance. type Maintainer interface { Maintain(ctx context.Context) error diff --git a/database/storage/sinkhole/sinkhole.go b/database/storage/sinkhole/sinkhole.go index d033638..af6b082 100644 --- a/database/storage/sinkhole/sinkhole.go +++ b/database/storage/sinkhole/sinkhole.go @@ -44,6 +44,11 @@ func (s *Sinkhole) Get(key string) (record.Record, error) { return nil, storage.ErrNotFound } +// GetMeta returns the metadata of a database record. +func (s *Sinkhole) GetMeta(key string) (*record.Meta, error) { + return nil, storage.ErrNotFound +} + // Put stores a record in the database. func (s *Sinkhole) Put(r record.Record) (record.Record, error) { return r, nil diff --git a/metrics/config.go b/metrics/config.go index 9767d76..dfcfc02 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -21,8 +21,8 @@ var ( ) func init() { - flag.StringVar(&pushFlag, "push-metrics", "", "Set default URL to push prometheus metrics to.") - flag.StringVar(&instanceFlag, "metrics-instance", "", "Set the default global instance label.") + flag.StringVar(&pushFlag, "push-metrics", "", "set default URL to push prometheus metrics to") + flag.StringVar(&instanceFlag, "metrics-instance", "", "set the default global instance label") } func prepConfig() error { diff --git a/modules/events.go b/modules/events.go index 54e70aa..fd1a9bf 100644 --- a/modules/events.go +++ b/modules/events.go @@ -6,8 +6,18 @@ import ( "fmt" "github.com/safing/portbase/log" + "github.com/tevino/abool" ) +type eventHooks struct { + // hooks holds all registed hooks for the event. + hooks []*eventHook + + // internal signifies that the event and it's data may not be exposed and may + // only be propagated internally. + internal bool +} + type eventHookFn func(context.Context, interface{}) error type eventHook struct { @@ -27,17 +37,26 @@ func (m *Module) processEventTrigger(event string, data interface{}) { m.eventHooksLock.RLock() defer m.eventHooksLock.RUnlock() - hooks, ok := m.eventHooks[event] + eventHooks, ok := m.eventHooks[event] if !ok { log.Warningf(`%s: tried to trigger non-existent event "%s"`, m.Name, event) return } - for _, hook := range hooks { + for _, hook := range eventHooks.hooks { if hook.hookingModule.OnlineSoon() { go m.runEventHook(hook, event, data) } } + + // Call subscription function, if set. + if eventSubscriptionFuncReady.IsSet() { + m.StartWorker("event subscription", func(context.Context) error { + // Only use data in worker that won't change anymore. + eventSubscriptionFunc(m.Name, event, eventHooks.internal, data) + return nil + }) + } } // InjectEvent triggers an event from a foreign module and executes all hook functions registered to that event. @@ -63,12 +82,21 @@ func (m *Module) InjectEvent(sourceEventName, targetModuleName, targetEventName return fmt.Errorf(`module "%s" has no event named "%s"`, targetModuleName, targetEventName) } - for _, hook := range targetHooks { + for _, hook := range targetHooks.hooks { if hook.hookingModule.OnlineSoon() { go m.runEventHook(hook, sourceEventName, data) } } + // Call subscription function, if set. + if eventSubscriptionFuncReady.IsSet() { + m.StartWorker("event subscription", func(context.Context) error { + // Only use data in worker that won't change anymore. + eventSubscriptionFunc(targetModule.Name, targetEventName, targetHooks.internal, data) + return nil + }) + } + return nil } @@ -111,13 +139,20 @@ func (m *Module) runEventHook(hook *eventHook, event string, data interface{}) { } // RegisterEvent registers a new event to allow for registering hooks. -func (m *Module) RegisterEvent(event string) { +// The expose argument controls whether these events and the attached data may +// be received by external components via APIs. If not exposed, the database +// record that carries the event and it's data will be marked as secret and as +// a crown jewel. Enforcement is left to the database layer. +func (m *Module) RegisterEvent(event string, expose bool) { m.eventHooksLock.Lock() defer m.eventHooksLock.Unlock() _, ok := m.eventHooks[event] if !ok { - m.eventHooks[event] = make([]*eventHook, 0, 1) + m.eventHooks[event] = &eventHooks{ + hooks: make([]*eventHook, 0, 1), + internal: !expose, + } } } @@ -138,16 +173,34 @@ func (m *Module) RegisterEventHook(module string, event string, description stri // get target event eventModule.eventHooksLock.Lock() defer eventModule.eventHooksLock.Unlock() - hooks, ok := eventModule.eventHooks[event] + eventHooks, ok := eventModule.eventHooks[event] if !ok { return fmt.Errorf(`event "%s/%s" does not exist`, eventModule.Name, event) } // add hook - eventModule.eventHooks[event] = append(hooks, &eventHook{ + eventHooks.hooks = append(eventHooks.hooks, &eventHook{ description: description, hookingModule: m, hookFn: fn, }) return nil } + +// Subscribe to all events + +var ( + eventSubscriptionFunc func(moduleName, eventName string, internal bool, data interface{}) + eventSubscriptionFuncEnabled = abool.NewBool(false) + eventSubscriptionFuncReady = abool.NewBool(false) +) + +// SetEventSubscriptionFunc +func SetEventSubscriptionFunc(fn func(moduleName, eventName string, internal bool, data interface{})) bool { + if eventSubscriptionFuncEnabled.SetToIf(false, true) { + eventSubscriptionFunc = fn + eventSubscriptionFuncReady.Set() + return true + } + return false +} diff --git a/modules/modules.go b/modules/modules.go index 286e3e2..2437a1e 100644 --- a/modules/modules.go +++ b/modules/modules.go @@ -40,6 +40,7 @@ type Module struct { //nolint:maligned // not worth the effort // failure status failureStatus uint8 failureID string + failureTitle string failureMsg string // lifecycle callback functions @@ -62,7 +63,7 @@ type Module struct { //nolint:maligned // not worth the effort waitGroup sync.WaitGroup // events - eventHooks map[string][]*eventHook + eventHooks map[string]*eventHooks eventHooksLock sync.RWMutex // dependency mgmt @@ -127,8 +128,9 @@ func (m *Module) prep(reports chan *report) { // set status if err != nil { m.Error( - "module-failed-prep", - fmt.Sprintf("failed to prep module: %s", err.Error()), + fmt.Sprintf("%s:prep-failed", m.Name), + fmt.Sprintf("Preparing module %s failed", m.Name), + fmt.Sprintf("Failed to prep module: %s", err.Error()), ) } else { m.Lock() @@ -183,8 +185,9 @@ func (m *Module) start(reports chan *report) { // set status if err != nil { m.Error( - "module-failed-start", - fmt.Sprintf("failed to start module: %s", err.Error()), + fmt.Sprintf("%s:start-failed", m.Name), + fmt.Sprintf("Starting module %s failed", m.Name), + fmt.Sprintf("Failed to start module: %s", err.Error()), ) } else { m.Lock() @@ -270,8 +273,9 @@ func (m *Module) stopAllTasks(reports chan *report) { // set status if err != nil { m.Error( - "module-failed-stop", - fmt.Sprintf("failed to stop module: %s", err.Error()), + fmt.Sprintf("%s:stop-failed", m.Name), + fmt.Sprintf("Stopping module %s failed", m.Name), + fmt.Sprintf("Failed to stop module: %s", err.Error()), ) } @@ -328,7 +332,7 @@ func initNewModule(name string, prep, start, stop func() error, dependencies ... taskCnt: &taskCnt, microTaskCnt: µTaskCnt, waitGroup: sync.WaitGroup{}, - eventHooks: make(map[string][]*eventHook), + eventHooks: make(map[string]*eventHooks), depNames: dependencies, } diff --git a/modules/status.go b/modules/status.go index 94ccc1b..fb7a93d 100644 --- a/modules/status.go +++ b/modules/status.go @@ -1,5 +1,11 @@ package modules +import ( + "context" + + "github.com/tevino/abool" +) + // Module Status Values const ( StatusDead uint8 = 0 // not prepared, not started @@ -25,6 +31,23 @@ const ( statusNothingToDo ) +var ( + failureUpdateNotifyFunc func(moduleFailure uint8, id, title, msg string) + failureUpdateNotifyFuncEnabled = abool.NewBool(false) + failureUpdateNotifyFuncReady = abool.NewBool(false) +) + +// SetFailureUpdateNotifyFunc sets a function that is called on every change +// of a module's failure status. +func SetFailureUpdateNotifyFunc(fn func(moduleFailure uint8, id, title, msg string)) bool { + if failureUpdateNotifyFuncEnabled.SetToIf(false, true) { + failureUpdateNotifyFunc = fn + failureUpdateNotifyFuncReady.Set() + return true + } + return false +} + // Online returns whether the module is online. func (m *Module) Online() bool { return m.Status() == StatusOnline @@ -56,40 +79,80 @@ func (m *Module) FailureStatus() (failureStatus uint8, failureID, failureMsg str return m.failureStatus, m.failureID, m.failureMsg } -// Hint sets failure status to hint. This is a somewhat special failure status, as the module is believed to be working correctly, but there is an important module specific information to convey. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. -func (m *Module) Hint(failureID, failureMsg string) { +// Hint sets failure status to hint. This is a somewhat special failure status, +// as the module is believed to be working correctly, but there is an important +// module specific information to convey. The supplied failureID is for +// improved automatic handling within connected systems, the failureMsg is for +// humans. +// The given ID must be unique for the given title and message. A call to +// Hint(), Warning() or Error() with the same ID as the existing one will be +// ignored. +func (m *Module) Hint(id, title, msg string) { m.Lock() defer m.Unlock() - m.failureStatus = FailureHint - m.failureID = failureID - m.failureMsg = failureMsg - - m.notifyOfChange() + m.setFailure(FailureHint, id, title, msg) } -// Warning sets failure status to warning. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. -func (m *Module) Warning(failureID, failureMsg string) { +// Warning sets failure status to warning. The supplied failureID is for +// improved automatic handling within connected systems, the failureMsg is for +// humans. +// The given ID must be unique for the given title and message. A call to +// Hint(), Warning() or Error() with the same ID as the existing one will be +// ignored. +func (m *Module) Warning(id, title, msg string) { m.Lock() defer m.Unlock() - m.failureStatus = FailureWarning - m.failureID = failureID - m.failureMsg = failureMsg - - m.notifyOfChange() + m.setFailure(FailureWarning, id, title, msg) } -// Error sets failure status to error. The supplied failureID is for improved automatic handling within connected systems, the failureMsg is for humans. -func (m *Module) Error(failureID, failureMsg string) { +// Error sets failure status to error. The supplied failureID is for improved +// automatic handling within connected systems, the failureMsg is for humans. +// The given ID must be unique for the given title and message. A call to +// Hint(), Warning() or Error() with the same ID as the existing one will be +// ignored. +func (m *Module) Error(id, title, msg string) { m.Lock() defer m.Unlock() - m.failureStatus = FailureError - m.failureID = failureID - m.failureMsg = failureMsg + m.setFailure(FailureError, id, title, msg) +} +func (m *Module) setFailure(status uint8, id, title, msg string) { + // Ignore calls with the same ID. + if id == m.failureID { + return + } + + // Copy data for failure status update worker. + resolveFailureID := m.failureID + + // Set new failure status. + m.failureStatus = status + m.failureID = id + m.failureTitle = title + m.failureMsg = msg + + // Notify of module change. m.notifyOfChange() + + // Propagate failure status. + if failureUpdateNotifyFuncReady.IsSet() { + m.newTask("failure status updater", func(context.Context, *Task) error { + // Only use data in worker that won't change anymore. + + // Resolve previous failure state if available. + if resolveFailureID != "" { + failureUpdateNotifyFunc(FailureNone, resolveFailureID, "", "") + } + + // Notify of new failure state. + failureUpdateNotifyFunc(status, id, title, msg) + + return nil + }).QueuePrioritized() + } } // Resolve removes the failure state from the module if the given failureID matches the current failure ID. If the given failureID is an empty string, Resolve removes any failure state. @@ -97,13 +160,33 @@ func (m *Module) Resolve(failureID string) { m.Lock() defer m.Unlock() - if failureID == "" || failureID == m.failureID { - m.failureStatus = FailureNone - m.failureID = "" - m.failureMsg = "" + // Check if resolving is necessary. + if failureID != "" && failureID != m.failureID { + // Return immediately if not resolving any (`""`) or if the failure ID + // does not match. + return } + // Copy data for failure status update worker. + resolveFailureID := m.failureID + + // Set failure status on module. + m.failureStatus = FailureNone + m.failureID = "" + m.failureTitle = "" + m.failureMsg = "" + + // Notify of module change. m.notifyOfChange() + + // Propagate failure status. + if failureUpdateNotifyFuncReady.IsSet() { + m.newTask("failure status updater", func(context.Context, *Task) error { + // Only use data in worker that won't change anymore. + failureUpdateNotifyFunc(FailureNone, resolveFailureID, "", "") + return nil + }).QueuePrioritized() + } } // readyToPrep returns whether all dependencies are ready for this module to prep. diff --git a/modules/subsystems/module.go b/modules/subsystems/module.go index 539fe77..48a3784 100644 --- a/modules/subsystems/module.go +++ b/modules/subsystems/module.go @@ -76,7 +76,8 @@ func prep() error { if err != nil { module.Error( "modulemgmt-failed", - fmt.Sprintf("The subsystem framework failed to start or stop one or more modules.\nError: %s\nCheck logs for more information.", err), + "A Module failed to start", + fmt.Sprintf("The subsystem framework failed to start or stop one or more modules.\nError: %s\nCheck logs for more information or try to restart.", err), ) return nil } diff --git a/modules/subsystems/subsystems_test.go b/modules/subsystems/subsystems_test.go index 3333df6..4351a34 100644 --- a/modules/subsystems/subsystems_test.go +++ b/modules/subsystems/subsystems_test.go @@ -77,7 +77,7 @@ func TestSubsystems(t *testing.T) { // test // let module fail - feature1.Error("test-fail", "Testing Fail") + feature1.Error("test-fail", "Test Fail", "Testing Fail") time.Sleep(10 * time.Millisecond) if sub1.FailureStatus != modules.FailureError { t.Fatal("error did not propagate") diff --git a/modules/tasks.go b/modules/tasks.go index 8c1bb93..1fbdec6 100644 --- a/modules/tasks.go +++ b/modules/tasks.go @@ -61,7 +61,10 @@ const ( defaultMaxDelay = 1 * time.Minute ) -// NewTask creates a new task with a descriptive name (non-unique), a optional deadline, and the task function to be executed. You must call one of Queue, Prioritize, StartASAP, Schedule or Repeat in order to have the Task executed. +// NewTask creates a new task with a descriptive name (non-unique), a optional +// deadline, and the task function to be executed. You must call one of Queue, +// QueuePrioritized, StartASAP, Schedule or Repeat in order to have the Task +// executed. func (m *Module) NewTask(name string, fn func(context.Context, *Task) error) *Task { if m == nil { log.Errorf(`modules: cannot create task "%s" with nil module`, name) @@ -75,6 +78,10 @@ func (m *Module) NewTask(name string, fn func(context.Context, *Task) error) *Ta m.Lock() defer m.Unlock() + return m.newTask(name, fn) +} + +func (m *Module) newTask(name string, fn func(context.Context, *Task) error) *Task { if m.Ctx == nil || !m.OnlineSoon() { log.Errorf(`modules: tasks should only be started when the module is online or starting`) return &Task{ @@ -144,8 +151,8 @@ func (t *Task) Queue() *Task { return t } -// Prioritize puts the task in the prioritized queue. -func (t *Task) Prioritize() *Task { +// QueuePrioritized queues the Task for execution in the prioritized queue. +func (t *Task) QueuePrioritized() *Task { t.lock.Lock() defer t.lock.Unlock() diff --git a/notifications/module-mirror.go b/notifications/module-mirror.go new file mode 100644 index 0000000..24f0741 --- /dev/null +++ b/notifications/module-mirror.go @@ -0,0 +1,107 @@ +package notifications + +import ( + "github.com/safing/portbase/log" + "github.com/safing/portbase/modules" +) + +// AttachToModule attaches the notification to a module and changes to the +// notification will be reflected on the module failure status. +func (n *Notification) AttachToModule(m *modules.Module) { + if m == nil { + log.Warningf("notifications: invalid usage: cannot attach %s to nil module", n.EventID) + return + } + + n.lock.Lock() + defer n.lock.Unlock() + + if n.State != Active { + log.Warningf("notifications: cannot attach module to inactive notification %s", n.EventID) + return + } + if n.belongsTo != nil { + log.Warningf("notifications: cannot override attached module for notification %s", n.EventID) + return + } + + // Attach module. + n.belongsTo = m + + // Set module failure status. + switch n.Type { //nolint:exhaustive + case Info: + m.Hint(n.EventID, n.Title, n.Message) + case Warning: + m.Warning(n.EventID, n.Title, n.Message) + case Error: + m.Error(n.EventID, n.Title, n.Message) + default: + log.Warningf("notifications: incompatible type for attaching to module in notification %s", n.EventID) + m.Error(n.EventID, n.Title, n.Message+" [incompatible notification type]") + } +} + +// resolveModuleFailure removes the notification from the module failure status. +func (n *Notification) resolveModuleFailure() { + if n.belongsTo != nil { + // Resolve failure in attached module. + n.belongsTo.Resolve(n.EventID) + + // Reset attachment in order to mitigate duplicate failure resolving. + // Re-attachment is prevented by the state check when attaching. + n.belongsTo = nil + } +} + +func init() { + modules.SetFailureUpdateNotifyFunc(mirrorModuleStatus) +} + +func mirrorModuleStatus(moduleFailure uint8, id, title, msg string) { + // Ignore "resolve all" requests. + if id == "" { + return + } + + // Get notification from storage. + n, ok := getNotification(id) + if ok { + // The notification already exists. + + // Check if we should delete it. + if moduleFailure == modules.FailureNone { + n.Delete() + } + + return + } + + // A notification for the given ID does not yet exists, create it. + n = &Notification{ + EventID: id, + Title: title, + Message: msg, + AvailableActions: []*Action{ + { + Text: "Get Help", + Type: ActionTypeOpenURL, + Payload: "https://safing.io/support/", + }, + }, + } + + switch moduleFailure { + case modules.FailureHint: + n.Type = Info + n.AvailableActions = nil + case modules.FailureWarning: + n.Type = Warning + n.ShowOnSystem = true + case modules.FailureError: + n.Type = Error + n.ShowOnSystem = true + } + + Notify(n) +} diff --git a/notifications/notification.go b/notifications/notification.go index 0c2a6be..0deef9d 100644 --- a/notifications/notification.go +++ b/notifications/notification.go @@ -8,6 +8,7 @@ import ( "github.com/safing/portbase/database/record" "github.com/safing/portbase/log" + "github.com/safing/portbase/modules" "github.com/safing/portbase/utils" ) @@ -19,6 +20,7 @@ const ( Info Type = 0 Warning Type = 1 Prompt Type = 2 + Error Type = 3 ) // State describes the state of a notification. @@ -70,6 +72,10 @@ type Notification struct { // of the notification is available. Note that the message should already // have any paramerized values replaced. Message string + // ShowOnSystem specifies if the notification should be also shown on the + // operating system. Notifications shown on the operating system level are + // more focus-intrusive and should only be used for important notifications. + ShowOnSystem bool // EventData contains an additional payload for the notification. This payload // may contain contextual data and may be used by a localization framework // to populate the notification message template. @@ -91,6 +97,10 @@ type Notification struct { // based on the user selection. SelectedActionID string + // belongsTo holds the module this notification belongs to. The notification + // lifecycle will be mirrored to the module's failure status. + belongsTo *modules.Module + lock sync.Mutex actionFunction NotificationActionFn // call function to process action actionTrigger chan string // and/or send to a channel @@ -99,8 +109,59 @@ type Notification struct { // Action describes an action that can be taken for a notification. type Action struct { - ID string + // ID specifies a unique ID for the action. If an action is selected, the ID + // is written to SelectedActionID and the notification is saved. + // If the action type is not ActionTypeNone, the ID may be empty, signifying + // that this action is merely additional and selecting it does not dismiss the + // notification. + ID string + // Text on the button. Text string + // Type specifies the action type. Implementing interfaces should only + // display action types they can handle. + Type ActionType + // Payload holds additional data for special action types. + Payload interface{} +} + +// ActionType defines a specific type of action. +type ActionType string + +// Action Types. +const ( + ActionTypeNone = "" // Report selected ID back to backend. + ActionTypeOpenURL = "open-url" // Open external URL + ActionTypeOpenPage = "open-page" // Payload: Page ID + ActionTypeOpenSetting = "open-setting" // Payload: See struct definition below. + ActionTypeOpenProfile = "open-profile" // Payload: Scoped Profile ID + ActionTypeInjectEvent = "inject-event" // Payload: Event ID + ActionTypeWebhook = "call-webhook" // Payload: See struct definition below. +) + +// ActionTypeOpenSettingPayload defines the payload for the OpenSetting Action Type. +type ActionTypeOpenSettingPayload struct { + // Key is the key of the setting. + Key string + // Profile is the scoped ID of the profile. + // Leaving this empty opens the global settings. + Profile string +} + +// ActionTypeWebhookPayload defines the payload for the WebhookPayload Action Type. +type ActionTypeWebhookPayload struct { + // HTTP Method to use. Defaults to "GET", or "POST" if a Payload is supplied. + Method string + // URL to call. + // If the URL is relative, prepend the current API endpoint base path. + // If the URL is absolute, send request to the Portmaster. + URL string + // Payload holds arbitrary payload data. + Payload interface{} + // ResultAction defines what should be done with successfully returned data. + // Must one of: + // - `ignore`: do nothing (default) + // - `display`: the result is a human readable message, display it in a success message. + ResultAction string } // Get returns the notification identifed by the given id or nil if it doesn't exist. @@ -114,38 +175,84 @@ func Get(id string) *Notification { return nil } -// NotifyInfo is a helper method for quickly showing a info -// notification. The notification is already shown. If id is -// an empty string a new UUIDv4 will be generated. -func NotifyInfo(id, msg string, actions ...Action) *Notification { - return notify(Info, id, msg, actions...) +// Delete deletes the notification with the given id. +func Delete(id string) { + // Delete notification in defer to enable deferred unlocking. + var n *Notification + var ok bool + defer func() { + if ok { + n.Delete() + } + }() + + notsLock.Lock() + defer notsLock.Unlock() + n, ok = nots[id] } -// NotifyWarn is a helper method for quickly showing a warning -// notification. The notification is already shown. If id is -// an empty string a new UUIDv4 will be generated. -func NotifyWarn(id, msg string, actions ...Action) *Notification { - return notify(Warning, id, msg, actions...) +// NotifyInfo is a helper method for quickly showing an info notification. +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. +// ShowOnSystem is disabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. +func NotifyInfo(id, title, msg string, actions ...Action) *Notification { + return notify(Info, id, title, msg, false, actions...) } -// NotifyPrompt is a helper method for quickly showing a prompt -// notification. The notification is already shown. If id is -// an empty string a new UUIDv4 will be generated. -func NotifyPrompt(id, msg string, actions ...Action) *Notification { - return notify(Prompt, id, msg, actions...) +// NotifyWarn is a helper method for quickly showing a warning notification +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. +// ShowOnSystem is enabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. +func NotifyWarn(id, title, msg string, actions ...Action) *Notification { + return notify(Warning, id, title, msg, true, actions...) } -func notify(nType Type, id, msg string, actions ...Action) *Notification { - acts := make([]*Action, len(actions)) - for idx := range actions { - a := actions[idx] - acts[idx] = &a +// NotifyError is a helper method for quickly showing an error notification. +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. +// ShowOnSystem is enabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. +func NotifyError(id, title, msg string, actions ...Action) *Notification { + return notify(Error, id, title, msg, true, actions...) +} + +// NotifyPrompt is a helper method for quickly showing a prompt notification. +// The notification will be activated immediately. +// If the provided id is empty, an id will derived from msg. +// ShowOnSystem is disabled. +// If no actions are defined, a default "OK" (ID:"ack") action will be added. +func NotifyPrompt(id, title, msg string, actions ...Action) *Notification { + return notify(Prompt, id, title, msg, false, actions...) +} + +func notify(nType Type, id, title, msg string, showOnSystem bool, actions ...Action) *Notification { + // Process actions. + var acts []*Action + if len(actions) == 0 { + // Create ack action if there are no defined actions. + acts = []*Action{ + { + ID: "ack", + Text: "OK", + }, + } + } else { + // Reference given actions for notification. + acts = make([]*Action, len(actions)) + for index := range actions { + a := actions[index] + acts[index] = &a + } } return Notify(&Notification{ EventID: id, Type: nType, + Title: title, Message: msg, + ShowOnSystem: showOnSystem, AvailableActions: acts, }) } @@ -185,15 +292,9 @@ func (n *Notification) save(pushUpdate bool) { n.lock.Lock() defer n.lock.Unlock() - // Move Title to Message, as that is the required field. - if n.Message == "" { - n.Message = n.Title - n.Title = "" - } - // Check if required data is present. - if n.Message == "" { - log.Warning("notifications: ignoring notification without Message") + if n.Title == "" && n.Message == "" { + log.Warning("notifications: ignoring notification without Title or Message") return } @@ -213,16 +314,6 @@ func (n *Notification) save(pushUpdate bool) { n.GUID = utils.RandomUUID(n.EventID).String() } - // Make ack notification if there are no defined actions. - if len(n.AvailableActions) == 0 { - n.AvailableActions = []*Action{ - { - ID: "ack", - Text: "OK", - }, - } - } - // Make sure we always have a notification state assigned. if n.State == "" { n.State = Active @@ -293,9 +384,8 @@ func (n *Notification) Update(expires int64) { } // Delete (prematurely) cancels and deletes a notification. -func (n *Notification) Delete() error { +func (n *Notification) Delete() { n.delete(true) - return nil } // delete deletes the notification from the internal storage. It locks the @@ -332,6 +422,8 @@ func (n *Notification) delete(pushUpdate bool) { if pushUpdate { dbController.PushUpdate(n) } + + n.resolveModuleFailure() } // Expired notifies the caller when the notification has expired. @@ -384,6 +476,7 @@ func (n *Notification) selectAndExecuteAction(id string) { if executed { n.State = Executed + n.resolveModuleFailure() } } diff --git a/runtime/module_api.go b/runtime/module.go similarity index 86% rename from runtime/module_api.go rename to runtime/module.go index 2316c91..2f22f61 100644 --- a/runtime/module_api.go +++ b/runtime/module.go @@ -1,6 +1,8 @@ package runtime import ( + "fmt" + "github.com/safing/portbase/database" "github.com/safing/portbase/modules" ) @@ -30,6 +32,10 @@ func startModule() error { return err } + if err := startModulesIntegration(); err != nil { + return fmt.Errorf("failed to start modules integration: %w", err) + } + return nil } diff --git a/runtime/modules_integration.go b/runtime/modules_integration.go new file mode 100644 index 0000000..2a16c47 --- /dev/null +++ b/runtime/modules_integration.go @@ -0,0 +1,72 @@ +package runtime + +import ( + "fmt" + "sync" + + "github.com/safing/portbase/database" + "github.com/safing/portbase/database/record" + "github.com/safing/portbase/log" + "github.com/safing/portbase/modules" +) + +var ( + modulesIntegrationUpdatePusher func(...record.Record) +) + +func startModulesIntegration() (err error) { + modulesIntegrationUpdatePusher, err = Register("modules/", &ModulesIntegration{}) + if err != nil { + return err + } + + if !modules.SetEventSubscriptionFunc(pushModuleEvent) { + log.Warningf("runtime: failed to register the modules event subscription function") + } + + return nil +} + +type ModulesIntegration struct{} + +// Set is called when the value is set from outside. +// If the runtime value is considered read-only ErrReadOnly +// should be returned. It is guaranteed that the key of +// the record passed to Set is prefixed with the key used +// to register the value provider. +func (mi *ModulesIntegration) Set(record.Record) (record.Record, error) { + return nil, ErrReadOnly +} + +// Get should return one or more records that match keyOrPrefix. +// keyOrPrefix is guaranteed to be at least the prefix used to +// register the ValueProvider. +func (mi *ModulesIntegration) Get(keyOrPrefix string) ([]record.Record, error) { + return nil, database.ErrNotFound +} + +type eventData struct { //nolint:unused // This is a cascading false positive. + record.Base + sync.Mutex + Data interface{} +} + +func pushModuleEvent(moduleName, eventName string, internal bool, data interface{}) { //nolint:unused // This is a false positive, the function is provided to modules.SetEventSubscriptionFunc(). + // Create event record and set key. + eventRecord := &eventData{ + Data: data, + } + eventRecord.SetKey(fmt.Sprintf( + "runtime:modules/%s/event/%s", + moduleName, + eventName, + )) + eventRecord.UpdateMeta() + if internal { + eventRecord.Meta().MakeSecret() + eventRecord.Meta().MakeCrownJewel() + } + + // Push event to database subscriptions. + modulesIntegrationUpdatePusher(eventRecord) +} diff --git a/template/module.go b/template/module.go index 72097ad..580b7fa 100644 --- a/template/module.go +++ b/template/module.go @@ -39,7 +39,7 @@ func init() { ) // register events that other modules can subscribe to - module.RegisterEvent(eventStateUpdate) + module.RegisterEvent(eventStateUpdate, true) } func prep() error { diff --git a/test b/test index d510075..ef569db 100755 --- a/test +++ b/test @@ -175,13 +175,16 @@ echo "running tests for ${platformInfo//$'\n'/ }:" # run vet/test on packages for package in $packages; do + package=${package#github.com/safing/portbase} + package=${package#/} + package=$PWD/$package echo "" echo $package if [[ $testonly -eq 0 ]]; then checkformat $package run golint -set_exit_status -min_confidence 1.0 $package run go vet $package - run golangci-lint run $GOPATH/src/$package + run golangci-lint run $package fi run go test -cover $fullTestFlags $package done diff --git a/updater/resource_test.go b/updater/resource_test.go index 2f734cc..038e4fe 100644 --- a/updater/resource_test.go +++ b/updater/resource_test.go @@ -36,14 +36,14 @@ func TestVersionSelection(t *testing.T) { registry.Beta = true registry.DevMode = true res.selectVersion() - if res.SelectedVersion.VersionNumber != "0" { - t.Errorf("selected version should be 0, not %s", res.SelectedVersion.VersionNumber) + if res.SelectedVersion.VersionNumber != "0.0.0" { + t.Errorf("selected version should be 0.0.0, not %s", res.SelectedVersion.VersionNumber) } registry.DevMode = false res.selectVersion() - if res.SelectedVersion.VersionNumber != "1.2.4b" { - t.Errorf("selected version should be 1.2.4b, not %s", res.SelectedVersion.VersionNumber) + if res.SelectedVersion.VersionNumber != "1.2.4-b" { + t.Errorf("selected version should be 1.2.4-b, not %s", res.SelectedVersion.VersionNumber) } registry.Beta = false