Merge pull request #111 from safing/feature/api-actions

Add endpoint api and authentication layer

api/auth_wrapper.go

@@ -0,0 +1,30 @@
+package api
+
+import "net/http"
+
+// WrapInAuthHandler wraps a simple http.HandlerFunc into a handler that
+// exposes the required API permissions for this handler.
+func WrapInAuthHandler(fn http.HandlerFunc, read, write Permission) http.Handler {
+	return &wrappedAuthenticatedHandler{
+		HandlerFunc: fn,
+		read:        read,
+		write:       write,
+	}
+}
+
+type wrappedAuthenticatedHandler struct {
+	http.HandlerFunc
+
+	read  Permission
+	write Permission
+}
+
+// ReadPermission returns the read permission for the handler.
+func (wah *wrappedAuthenticatedHandler) ReadPermission(r *http.Request) Permission {
+	return wah.read
+}
+
+// WritePermission returns the write permission for the handler.
+func (wah *wrappedAuthenticatedHandler) WritePermission(r *http.Request) Permission {
+	return wah.write
+}

api/authentication.go

@@ -8,43 +8,114 @@ import (
 	"sync"
 	"time"
 
+	"github.com/tevino/abool"
+
 	"github.com/safing/portbase/modules"
 
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/rng"
 )
 
-var (
-	validTokens     = make(map[string]time.Time)
-	validTokensLock sync.Mutex
+// Permission defines an API requests permission.
+type Permission int8
 
-	authFnLock sync.Mutex
-	authFn     Authenticator
+const (
+	// NotFound declares that the operation does not exist.
+	NotFound Permission = -2
 
-	// ErrAPIAccessDeniedMessage should be returned by Authenticator functions in
-	// order to signify a blocked request, including a error message for the user.
-	ErrAPIAccessDeniedMessage = errors.New("")
+	// Require declares that the operation requires permission to be processed,
+	// but anyone can execute the operation.
+	Require Permission = -1
+
+	// NotSupported declares that the operation is not supported.
+	NotSupported Permission = 0
+
+	// PermitAnyone declares that anyone can execute the operation without any
+	// authentication.
+	PermitAnyone Permission = 1
+
+	// PermitUser declares that the operation may be executed by authenticated
+	// third party applications that are categorized as representing a simple
+	// user and is limited in access.
+	PermitUser Permission = 2
+
+	// PermitAdmin declares that the operation may be executed by authenticated
+	// third party applications that are categorized as representing an
+	// administrator and has broad in access.
+	PermitAdmin Permission = 3
+
+	// PermitSelf declares that the operation may only be executed by the
+	// software itself and its own (first party) components.
+	PermitSelf Permission = 4
 )
 
+// AuthenticatorFunc is a function that can be set as the authenticator for the
+// API endpoint. If none is set, all requests will have full access.
+// The returned AuthToken represents the permissions that the request has.
+type AuthenticatorFunc func(r *http.Request, s *http.Server) (*AuthToken, error)
+
+// AuthToken represents either a set of required or granted permissions.
+// All attributes must be set when the struct is built and must not be changed
+// later. Functions may be called at any time.
+// The Write permission implicitly also includes reading.
+type AuthToken struct {
+	Read  Permission
+	Write Permission
+
+	validUntil time.Time
+	validLock  sync.Mutex
+}
+
+// Expired returns whether the token has expired.
+func (token *AuthToken) Expired() bool {
+	token.validLock.Lock()
+	defer token.validLock.Unlock()
+
+	return time.Now().After(token.validUntil)
+}
+
+// Refresh refreshes the validity of the token with the given TTL.
+func (token *AuthToken) Refresh(ttl time.Duration) {
+	token.validLock.Lock()
+	defer token.validLock.Unlock()
+
+	token.validUntil = time.Now().Add(ttl)
+}
+
+// AuthenticatedHandler defines the handler interface to specify custom
+// permission for an API handler. The returned permission is the required
+// permission for the request to proceed.
+type AuthenticatedHandler interface {
+	ReadPermission(*http.Request) Permission
+	WritePermission(*http.Request) Permission
+}
+
 const (
 	cookieName = "Portmaster-API-Token"
-
-	cookieTTL = 5 * time.Minute
+	cookieTTL  = 5 * time.Minute
 )
 
-// Authenticator is a function that can be set as the authenticator for the API endpoint. If none is set, all requests will be permitted.
-type Authenticator func(ctx context.Context, s *http.Server, r *http.Request) (err error)
+var (
+	authFnSet = abool.New()
+	authFn    AuthenticatorFunc
+
+	authTokens     = make(map[string]*AuthToken)
+	authTokensLock sync.Mutex
+
+	// ErrAPIAccessDeniedMessage should be wrapped by errors returned by
+	// AuthenticatorFunc in order to signify a blocked request, including a error
+	// message for the user. This is an empty message on purpose, as to allow the
+	// function to define the full text of the error shown to the user.
+	ErrAPIAccessDeniedMessage = errors.New("")
+)
 
 // SetAuthenticator sets an authenticator function for the API endpoint. If none is set, all requests will be permitted.
-func SetAuthenticator(fn Authenticator) error {
+func SetAuthenticator(fn AuthenticatorFunc) error {
 	if module.Online() {
 		return ErrAuthenticationImmutable
 	}
 
-	authFnLock.Lock()
-	defer authFnLock.Unlock()
-
-	if authFn != nil {
+	if !authFnSet.SetToIf(false, true) {
 		return ErrAuthenticationAlreadySet
 	}
 
@@ -54,92 +125,239 @@ func SetAuthenticator(fn Authenticator) error {
 
 func authMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		tracer := log.Tracer(r.Context())
-
-		// get authenticator
-		authFnLock.Lock()
-		authenticator := authFn
-		authFnLock.Unlock()
-
-		// permit if no authenticator set
-		if authenticator == nil {
+		token := authenticateRequest(w, r, next)
+		if token != nil {
+			if _, apiRequest := getAPIContext(r); apiRequest != nil {
+				apiRequest.AuthToken = token
+			}
 			next.ServeHTTP(w, r)
-			return
 		}
-
-		// check existing auth cookie
-		c, err := r.Cookie(cookieName)
-		if err == nil {
-			// get token
-			validTokensLock.Lock()
-			validUntil, valid := validTokens[c.Value]
-			validTokensLock.Unlock()
-
-			// check if token is valid
-			if valid && time.Now().Before(validUntil) {
-				tracer.Tracef("api: auth token %s is valid, refreshing", c.Value)
-				// refresh cookie
-				validTokensLock.Lock()
-				validTokens[c.Value] = time.Now().Add(cookieTTL)
-				validTokensLock.Unlock()
-				// continue
-				next.ServeHTTP(w, r)
-				return
-			}
-
-			tracer.Tracef("api: provided auth token %s is invalid", c.Value)
-		}
-
-		// get auth decision
-		err = authenticator(r.Context(), server, r)
-		if err != nil {
-			if errors.Is(err, ErrAPIAccessDeniedMessage) {
-				tracer.Warningf("api: denying api access to %s", r.RemoteAddr)
-				http.Error(w, err.Error(), http.StatusForbidden)
-			} else {
-				tracer.Warningf("api: authenticator failed: %s", err)
-				http.Error(w, "Internal server error during authentication.", http.StatusInternalServerError)
-			}
-			return
-		}
-
-		// generate new token
-		token, err := rng.Bytes(32) // 256 bit
-		if err != nil {
-			tracer.Warningf("api: failed to generate random token: %s", err)
-			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		}
-		tokenString := base64.RawURLEncoding.EncodeToString(token)
-		// write new cookie
-		http.SetCookie(w, &http.Cookie{
-			Name:     cookieName,
-			Value:    tokenString,
-			Path:     "/",
-			HttpOnly: true,
-			SameSite: http.SameSiteStrictMode,
-			MaxAge:   int(cookieTTL.Seconds()),
-		})
-		// save cookie
-		validTokensLock.Lock()
-		validTokens[tokenString] = time.Now().Add(cookieTTL)
-		validTokensLock.Unlock()
-
-		// serve
-		tracer.Tracef("api: granted %s, assigned auth token %s", r.RemoteAddr, tokenString)
-		next.ServeHTTP(w, r)
 	})
 }
 
-func cleanAuthTokens(_ context.Context, _ *modules.Task) error {
-	validTokensLock.Lock()
-	defer validTokensLock.Unlock()
+func authenticateRequest(w http.ResponseWriter, r *http.Request, targetHandler http.Handler) *AuthToken {
+	tracer := log.Tracer(r.Context())
 
-	now := time.Now()
-	for token, validUntil := range validTokens {
-		if now.After(validUntil) {
-			delete(validTokens, token)
+	// Check if authenticator is set.
+	if !authFnSet.IsSet() {
+		// Return highest available permissions for the request.
+		return &AuthToken{
+			Read:  PermitSelf,
+			Write: PermitSelf,
+		}
+	}
+
+	// Check if request is read only.
+	readRequest := isReadMethod(r.Method)
+
+	// Get required permission for target handler.
+	requiredPermission := PermitSelf
+	if authdHandler, ok := targetHandler.(AuthenticatedHandler); ok {
+		if readRequest {
+			requiredPermission = authdHandler.ReadPermission(r)
+		} else {
+			requiredPermission = authdHandler.WritePermission(r)
+		}
+	}
+
+	// Check if we need to do any authentication at all.
+	switch requiredPermission {
+	case NotFound:
+		// Not found.
+		tracer.Trace("api: authenticated handler reported: not found")
+		http.Error(w, "Not found.", http.StatusNotFound)
+		return nil
+	case NotSupported:
+		// A read or write permission can be marked as not supported.
+		tracer.Trace("api: authenticated handler reported: not supported")
+		http.Error(w, "Method not allowed.", http.StatusMethodNotAllowed)
+		return nil
+	case PermitAnyone:
+		// Don't process permissions, as we don't need them.
+		tracer.Tracef("api: granted %s access to public handler", r.RemoteAddr)
+		return &AuthToken{
+			Read:  PermitAnyone,
+			Write: PermitAnyone,
+		}
+	case Require:
+		// Continue processing permissions, but treat as PermitAnyone.
+		requiredPermission = PermitAnyone
+	}
+
+	// The required permission must match the request permission values after
+	// handling the specials.
+	if requiredPermission < PermitAnyone || requiredPermission > PermitSelf {
+		tracer.Warningf(
+			"api: handler returned invalid permission: %s (%d)",
+			requiredPermission,
+			requiredPermission,
+		)
+		http.Error(w, "Internal server error during authentication.", http.StatusInternalServerError)
+		return nil
+	}
+
+	// Check for an existing auth token.
+	token := checkAuthToken(r)
+
+	// Get auth token from authenticator if none was in the request.
+	if token == nil {
+		var err error
+		token, err = authFn(r, server)
+		if err != nil {
+			// Check for internal error.
+			if !errors.Is(err, ErrAPIAccessDeniedMessage) {
+				tracer.Errorf("api: authenticator failed: %s", err)
+				http.Error(w, "Internal server error during authentication.", http.StatusInternalServerError)
+				return nil
+			}
+
+			// If authentication failed and we require authentication, return an
+			// authentication error.
+			if requiredPermission != PermitAnyone {
+				// Return authentication error.
+				tracer.Warningf("api: denying api access to %s", r.RemoteAddr)
+				http.Error(w, err.Error(), http.StatusForbidden)
+				return nil
+			}
+
+			token = &AuthToken{
+				Read:  PermitAnyone,
+				Write: PermitAnyone,
+			}
+		}
+
+		// Apply auth token to request.
+		err = applyAuthToken(w, token)
+		if err != nil {
+			tracer.Warningf("api: failed to create auth token: %s", err)
+		}
+	}
+
+	// Get effective permission for request.
+	var requestPermission Permission
+	if readRequest {
+		requestPermission = token.Read
+	} else {
+		requestPermission = token.Write
+	}
+
+	// Check for valid request permission.
+	if requestPermission < PermitAnyone || requestPermission > PermitSelf {
+		tracer.Warningf(
+			"api: authenticator returned invalid permission: %s (%d)",
+			requestPermission,
+			requestPermission,
+		)
+		http.Error(w, "Internal server error during authentication.", http.StatusInternalServerError)
+		return nil
+	}
+
+	// Check permission.
+	if requestPermission < requiredPermission {
+		http.Error(w, "Insufficient permissions.", http.StatusForbidden)
+		return nil
+	}
+
+	tracer.Tracef("api: granted %s access to authenticated handler", r.RemoteAddr)
+
+	// Make a copy of the AuthToken in order mitigate the handler poisoning the
+	// token, as changes would apply to future requests.
+	return &AuthToken{
+		Read:  token.Read,
+		Write: token.Write,
+	}
+}
+
+func checkAuthToken(r *http.Request) *AuthToken {
+	// Get auth token from request.
+	c, err := r.Cookie(cookieName)
+	if err != nil {
+		return nil
+	}
+
+	// Check if auth token is registered.
+	authTokensLock.Lock()
+	token, ok := authTokens[c.Value]
+	authTokensLock.Unlock()
+	if !ok {
+		log.Tracer(r.Context()).Tracef("api: provided auth token %s is unknown", c.Value)
+		return nil
+	}
+
+	// Check if token is still valid.
+	if token.Expired() {
+		log.Tracer(r.Context()).Tracef("api: provided auth token %s has expired", c.Value)
+		return nil
+	}
+
+	// Refresh token and return.
+	token.Refresh(cookieTTL)
+	log.Tracer(r.Context()).Tracef("api: auth token %s is valid, refreshing", c.Value)
+	return token
+}
+
+func applyAuthToken(w http.ResponseWriter, token *AuthToken) error {
+	// Generate new token secret.
+	secret, err := rng.Bytes(32) // 256 bit
+	if err != nil {
+		return err
+	}
+	secretHex := base64.RawURLEncoding.EncodeToString(secret)
+
+	// Set token cookie in response.
+	http.SetCookie(w, &http.Cookie{
+		Name:     cookieName,
+		Value:    secretHex,
+		Path:     "/",
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+	})
+
+	// Set token TTL.
+	token.Refresh(cookieTTL)
+
+	// Save token.
+	authTokensLock.Lock()
+	defer authTokensLock.Unlock()
+	authTokens[secretHex] = token
+
+	return nil
+}
+
+func cleanAuthTokens(_ context.Context, _ *modules.Task) error {
+	authTokensLock.Lock()
+	defer authTokensLock.Unlock()
+
+	for secret, token := range authTokens {
+		if token.Expired() {
+			delete(authTokens, secret)
 		}
 	}
 
 	return nil
 }
+
+func isReadMethod(method string) bool {
+	return method == http.MethodGet || method == http.MethodHead
+}
+
+func (p Permission) String() string {
+	switch p {
+	case NotSupported:
+		return "NotSupported"
+	case Require:
+		return "Require"
+	case PermitAnyone:
+		return "PermitAnyone"
+	case PermitUser:
+		return "PermitUser"
+	case PermitAdmin:
+		return "PermitAdmin"
+	case PermitSelf:
+		return "PermitSelf"
+	case NotFound:
+		return "NotFound"
+	default:
+		return "Unknown"
+	}
+}

api/authentication_test.go

@@ -0,0 +1,195 @@
+package api
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	testToken = new(AuthToken)
+)
+
+func testAuthenticator(r *http.Request, s *http.Server) (*AuthToken, error) {
+	switch {
+	case testToken.Read == -127 || testToken.Write == -127:
+		return nil, errors.New("test error")
+	case testToken.Read == -128 || testToken.Write == -128:
+		return nil, fmt.Errorf("%wdenied", ErrAPIAccessDeniedMessage)
+	default:
+		return testToken, nil
+	}
+}
+
+type testAuthHandler struct {
+	Read  Permission
+	Write Permission
+}
+
+func (ah *testAuthHandler) ReadPermission(r *http.Request) Permission {
+	return ah.Read
+}
+
+func (ah *testAuthHandler) WritePermission(r *http.Request) Permission {
+	return ah.Write
+}
+
+func (ah *testAuthHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Check if request is as expected.
+	ar := GetAPIRequest(r)
+	switch {
+	case ar == nil:
+		http.Error(w, "ar == nil", http.StatusInternalServerError)
+	case ar.AuthToken == nil:
+		http.Error(w, "ar.AuthToken == nil", http.StatusInternalServerError)
+	default:
+		http.Error(w, "auth success", http.StatusOK)
+	}
+}
+
+func makeAuthTestPath(reading bool, p Permission) string {
+	if reading {
+		return fmt.Sprintf("/test/auth/read/%s", p)
+	}
+	return fmt.Sprintf("/test/auth/write/%s", p)
+}
+
+func init() {
+	// Set test authenticator.
+	err := SetAuthenticator(testAuthenticator)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestPermissions(t *testing.T) { //nolint:gocognit
+	testHandler := &mainHandler{
+		mux: mainMux,
+	}
+
+	// Define permissions that need testing.
+	permissionsToTest := []Permission{
+		NotSupported,
+		PermitAnyone,
+		PermitUser,
+		PermitAdmin,
+		PermitSelf,
+		Require,
+		NotFound,
+		100,  // Test a too high value.
+		-100, // Test a too low value.
+		-127, // Simulate authenticator failure.
+		-128, // Simulate authentication denied message.
+	}
+
+	// Register test handlers.
+	for _, p := range permissionsToTest {
+		RegisterHandler(makeAuthTestPath(true, p), &testAuthHandler{Read: p})
+		RegisterHandler(makeAuthTestPath(false, p), &testAuthHandler{Write: p})
+	}
+
+	// Test all the combinations.
+	for _, requestPerm := range permissionsToTest {
+		for _, handlerPerm := range permissionsToTest {
+			for _, method := range []string{
+				http.MethodGet,
+				http.MethodHead,
+				http.MethodPost,
+				http.MethodPut,
+			} {
+
+				// Set request permission for test requests.
+				reading := isReadMethod(method)
+				if reading {
+					testToken.Read = requestPerm
+					testToken.Write = NotSupported
+				} else {
+					testToken.Read = NotSupported
+					testToken.Write = requestPerm
+				}
+
+				// Evaluate expected result.
+				var expectSuccess bool
+				switch {
+				case handlerPerm == PermitAnyone:
+					// This is fast-tracked. There are not additional checks.
+					expectSuccess = true
+				case handlerPerm == Require:
+					// This is turned into PermitAnyone in the authenticator.
+					// But authentication is still processed and the result still gets
+					// sanity checked!
+					if requestPerm >= PermitAnyone &&
+						requestPerm <= PermitSelf {
+						expectSuccess = true
+					}
+					// Another special case is when the handler requires permission to be
+					// processed but the authenticator fails to authenticate the request.
+					// In this case, a fallback token with PermitAnyone is used.
+					if requestPerm == -128 {
+						// -128 is used to simulate a permission denied message.
+						expectSuccess = true
+					}
+				case handlerPerm <= NotSupported:
+					// Invalid handler permission.
+				case handlerPerm > PermitSelf:
+					// Invalid handler permission.
+				case requestPerm <= NotSupported:
+					// Invalid request permission.
+				case requestPerm > PermitSelf:
+					// Invalid request permission.
+				case requestPerm < handlerPerm:
+					// Valid, but insufficient request permission.
+				default:
+					expectSuccess = true
+				}
+
+				if expectSuccess {
+
+					// Test for success.
+					if !assert.HTTPBodyContains(
+						t,
+						testHandler.ServeHTTP,
+						method,
+						makeAuthTestPath(reading, handlerPerm),
+						nil,
+						"auth success",
+					) {
+						t.Errorf(
+							"%s with %s (%d) to handler %s (%d)",
+							method,
+							requestPerm, requestPerm,
+							handlerPerm, handlerPerm,
+						)
+					}
+
+				} else {
+
+					// Test for error.
+					if !assert.HTTPError(t,
+						testHandler.ServeHTTP,
+						method,
+						makeAuthTestPath(reading, handlerPerm),
+						nil,
+					) {
+						t.Errorf(
+							"%s with %s (%d) to handler %s (%d)",
+							method,
+							requestPerm, requestPerm,
+							handlerPerm, handlerPerm,
+						)
+					}
+
+				}
+			}
+		}
+	}
+}
+
+func TestPermissionDefinitions(t *testing.T) {
+	if NotSupported != 0 {
+		t.Fatalf("NotSupported must be zero, was %v", NotSupported)
+	}
+}

api/database.go

@@ -278,7 +278,7 @@ func (api *DatabaseAPI) handleGet(opID []byte, key string) {
 
 	r, err := api.db.Get(key)
 	if err == nil {
-		data, err = marshalRecord(r)
+		data, err = marshalRecord(r, true)
 	}
 	if err != nil {
 		api.send(opID, dbMsgTypeError, err.Error(), nil)
@@ -336,7 +336,7 @@ func (api *DatabaseAPI) processQuery(opID []byte, q *query.Query) (ok bool) {
 			// process query feed
 			if r != nil {
 				// process record
-				data, err := marshalRecord(r)
+				data, err := marshalRecord(r, true)
 				if err != nil {
 					api.send(opID, dbMsgTypeWarning, err.Error(), nil)
 				}
@@ -412,7 +412,7 @@ func (api *DatabaseAPI) processSub(opID []byte, sub *database.Subscription) {
 			// process sub feed
 			if r != nil {
 				// process record
-				data, err := marshalRecord(r)
+				data, err := marshalRecord(r, true)
 				if err != nil {
 					api.send(opID, dbMsgTypeWarning, err.Error(), nil)
 					continue
@@ -625,7 +625,7 @@ func (api *DatabaseAPI) shutdown() {
 
 // marsharlRecords locks and marshals the given record, additionally adding
 // metadata and returning it as json.
-func marshalRecord(r record.Record) ([]byte, error) {
+func marshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) {
 	r.Lock()
 	defer r.Unlock()
 
@@ -651,10 +651,12 @@ func marshalRecord(r record.Record) ([]byte, error) {
 	}
 
 	// Add JSON identifier again.
-	formatID := varint.Pack8(record.JSON)
-	finalData := make([]byte, 0, len(formatID)+len(jsonData))
-	finalData = append(finalData, formatID...)
-	finalData = append(finalData, jsonData...)
-
-	return finalData, nil
+	if withDSDIdentifier {
+		formatID := varint.Pack8(record.JSON)
+		finalData := make([]byte, 0, len(formatID)+len(jsonData))
+		finalData = append(finalData, formatID...)
+		finalData = append(finalData, jsonData...)
+		return finalData, nil
+	}
+	return jsonData, nil
 }

api/doc.go

@@ -0,0 +1,10 @@
+/*
+Package api provides an API for integration with other components of the same software package and also third party components.
+
+It provides direct database access as well as a simpler way to register API endpoints. You can of course also register raw `http.Handler`s directly.
+
+Optional authentication guards registered handlers. This is achieved by attaching functions to the `http.Handler`s that are registered, which allow them to specify the required permissions for the handler.
+
+The permissions are divided into the roles and assume a single user per host. The Roles are User, Admin and Self. User roles are expected to have mostly read access and react to notifications or system events, like a system tray program. The Admin role is meant for advanced components that also change settings, but are restricted so they cannot break the software. Self is reserved for internal use with full access.
+*/
+package api

api/endpoints.go

@@ -0,0 +1,302 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/log"
+)
+
+// Endpoint describes an API Endpoint.
+// Path and at least one permission are required.
+// As is exactly one function.
+type Endpoint struct {
+	Path     string
+	MimeType string
+	Read     Permission
+	Write    Permission
+
+	// TODO: We _could_ expose more metadata to be able to build lists of actions
+	// automatically.
+	// Name           string
+	// Description    string
+	// Order          int
+	// ExpertiseLevel config.ExpertiseLevel
+
+	// ActionFunc is for simple actions with a return message for the user.
+	ActionFunc ActionFunc `json:"-"`
+
+	// DataFunc is for returning raw data that the caller for further processing.
+	DataFunc DataFunc `json:"-"`
+
+	// StructFunc is for returning any kind of struct.
+	StructFunc StructFunc `json:"-"`
+
+	// RecordFunc is for returning a database record. It will be properly locked
+	// and marshalled including metadata.
+	RecordFunc RecordFunc `json:"-"`
+
+	// HandlerFunc is the raw http handler.
+	HandlerFunc http.HandlerFunc `json:"-"`
+}
+
+type (
+	// ActionFunc is for simple actions with a return message for the user.
+	ActionFunc func(ar *Request) (msg string, err error)
+
+	// DataFunc is for returning raw data that the caller for further processing.
+	DataFunc func(ar *Request) (data []byte, err error)
+
+	// StructFunc is for returning any kind of struct.
+	StructFunc func(ar *Request) (i interface{}, err error)
+
+	// RecordFunc is for returning a database record. It will be properly locked
+	// and marshalled including metadata.
+	RecordFunc func(ar *Request) (r record.Record, err error)
+)
+
+// MIME Types
+const (
+	MimeTypeJSON string = "application/json"
+	MimeTypeText string = "text/plain"
+
+	apiV1Path = "/api/v1/"
+)
+
+func init() {
+	RegisterHandler(apiV1Path+"{endpointPath:.+}", &endpointHandler{})
+}
+
+var (
+	endpoints     = make(map[string]*Endpoint)
+	endpointsLock sync.RWMutex
+
+	// ErrInvalidEndpoint is returned when an invalid endpoint is registered.
+	ErrInvalidEndpoint = errors.New("endpoint is invalid")
+
+	// ErrAlreadyRegistered is returned when there already is an endpoint with
+	// the same path registered.
+	ErrAlreadyRegistered = errors.New("an endpoint for this path is already registered")
+)
+
+func getAPIContext(r *http.Request) (apiEndpoint *Endpoint, apiRequest *Request) {
+	// Get request context and check if we already have an action cached.
+	apiRequest = GetAPIRequest(r)
+	if apiRequest == nil {
+		return nil, nil
+	}
+	var ok bool
+	apiEndpoint, ok = apiRequest.HandlerCache.(*Endpoint)
+	if ok {
+		return apiEndpoint, apiRequest
+	}
+
+	// If not, get the action from the registry.
+	endpointPath, ok := apiRequest.URLVars["endpointPath"]
+	if !ok {
+		return nil, apiRequest
+	}
+
+	endpointsLock.RLock()
+	defer endpointsLock.RUnlock()
+
+	apiEndpoint, ok = endpoints[endpointPath]
+	if ok {
+		// Cache for next operation.
+		apiRequest.HandlerCache = apiEndpoint
+	}
+	return apiEndpoint, apiRequest
+}
+
+// RegisterEndpoint registers a new endpoint. An error will be returned if it
+// does not pass the sanity checks.
+func RegisterEndpoint(e Endpoint) error {
+	if err := e.check(); err != nil {
+		return fmt.Errorf("%w: %s", ErrInvalidEndpoint, err)
+	}
+
+	endpointsLock.Lock()
+	defer endpointsLock.Unlock()
+
+	_, ok := endpoints[e.Path]
+	if ok {
+		return ErrAlreadyRegistered
+	}
+
+	endpoints[e.Path] = &e
+	return nil
+}
+
+func (e *Endpoint) check() error {
+	// Check path.
+	if strings.TrimSpace(e.Path) == "" {
+		return errors.New("path is missing")
+	}
+
+	// Check permissions.
+	if e.Read < Require || e.Read > PermitSelf {
+		return errors.New("invalid read permission")
+	}
+	if e.Write < Require || e.Write > PermitSelf {
+		return errors.New("invalid write permission")
+	}
+
+	// Check functions.
+	var defaultMimeType string
+	fnCnt := 0
+	if e.ActionFunc != nil {
+		fnCnt++
+		defaultMimeType = MimeTypeText
+	}
+	if e.DataFunc != nil {
+		fnCnt++
+		defaultMimeType = MimeTypeText
+	}
+	if e.StructFunc != nil {
+		fnCnt++
+		defaultMimeType = MimeTypeJSON
+	}
+	if e.RecordFunc != nil {
+		fnCnt++
+		defaultMimeType = MimeTypeJSON
+	}
+	if e.HandlerFunc != nil {
+		fnCnt++
+		defaultMimeType = MimeTypeText
+	}
+	if fnCnt != 1 {
+		return errors.New("only one function may be set")
+	}
+
+	// Set default mime type.
+	if e.MimeType == "" {
+		e.MimeType = defaultMimeType
+	}
+
+	return nil
+}
+
+type endpointHandler struct{}
+
+var _ AuthenticatedHandler = &endpointHandler{} // Compile time interface check.
+
+// ReadPermission returns the read permission for the handler.
+func (eh *endpointHandler) ReadPermission(r *http.Request) Permission {
+	apiEndpoint, _ := getAPIContext(r)
+	if apiEndpoint != nil {
+		return apiEndpoint.Read
+	}
+	return NotFound
+}
+
+// WritePermission returns the write permission for the handler.
+func (eh *endpointHandler) WritePermission(r *http.Request) Permission {
+	apiEndpoint, _ := getAPIContext(r)
+	if apiEndpoint != nil {
+		return apiEndpoint.Write
+	}
+	return NotFound
+}
+
+// ServeHTTP handles the http request.
+func (eh *endpointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	apiEndpoint, apiRequest := getAPIContext(r)
+	if apiEndpoint == nil || apiRequest == nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	switch r.Method {
+	case http.MethodHead:
+		w.WriteHeader(http.StatusOK)
+		return
+	case http.MethodPost, http.MethodPut:
+		// Read body data.
+		inputData, ok := readBody(w, r)
+		if !ok {
+			return
+		}
+		apiRequest.InputData = inputData
+	case http.MethodGet:
+		// Nothing special to do here.
+	default:
+		http.Error(w, "Unsupported method for the actions API.", http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Execute action function and get response data
+	var responseData []byte
+	var err error
+
+	switch {
+	case apiEndpoint.ActionFunc != nil:
+		var msg string
+		msg, err = apiEndpoint.ActionFunc(apiRequest)
+		if err == nil {
+			responseData = []byte(msg)
+		}
+
+	case apiEndpoint.DataFunc != nil:
+		responseData, err = apiEndpoint.DataFunc(apiRequest)
+
+	case apiEndpoint.StructFunc != nil:
+		var v interface{}
+		v, err = apiEndpoint.StructFunc(apiRequest)
+		if err == nil && v != nil {
+			responseData, err = json.Marshal(v)
+		}
+
+	case apiEndpoint.RecordFunc != nil:
+		var rec record.Record
+		rec, err = apiEndpoint.RecordFunc(apiRequest)
+		if err == nil && r != nil {
+			responseData, err = marshalRecord(rec, false)
+		}
+
+	case apiEndpoint.HandlerFunc != nil:
+		apiEndpoint.HandlerFunc(w, r)
+		return
+
+	default:
+		http.Error(w, "Internal server error: Missing handler.", http.StatusInternalServerError)
+		return
+	}
+
+	// Check for handler error.
+	if err != nil {
+		http.Error(w, "Internal server error: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Write response.
+	w.Header().Set("Content-Type", apiEndpoint.MimeType+"; charset=utf-8")
+	w.Header().Set("Content-Length", strconv.Itoa(len(responseData)))
+	w.WriteHeader(http.StatusOK)
+	_, err = w.Write(responseData)
+	if err != nil {
+		log.Tracer(r.Context()).Warningf("api: failed to write response: %s", err)
+	}
+}
+
+func readBody(w http.ResponseWriter, r *http.Request) (inputData []byte, ok bool) {
+	// Check for too long content in order to prevent death.
+	if r.ContentLength > 20000000 { // 20MB
+		http.Error(w, "Too much input data.", http.StatusRequestEntityTooLarge)
+		return nil, false
+	}
+
+	// Read and close body.
+	inputData, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "Failed to read body: "+err.Error(), http.StatusInternalServerError)
+		return nil, false
+	}
+	return inputData, true
+}

api/endpoints_debug.go

@@ -0,0 +1,80 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"runtime/pprof"
+
+	"github.com/safing/portbase/utils/osdetail"
+)
+
+func registerDebugEndpoints() error {
+	if err := RegisterEndpoint(Endpoint{
+		Path:     "debug/stack",
+		Read:     PermitAnyone,
+		DataFunc: getStack,
+	}); err != nil {
+		return err
+	}
+
+	if err := RegisterEndpoint(Endpoint{
+		Path:       "debug/stack/print",
+		Read:       PermitAnyone,
+		ActionFunc: printStack,
+	}); err != nil {
+		return err
+	}
+
+	if err := RegisterEndpoint(Endpoint{
+		Path:     "debug/info",
+		Read:     PermitAnyone,
+		DataFunc: debugInfo,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// getStack returns the current goroutine stack.
+func getStack(_ *Request) (data []byte, err error) {
+	buf := &bytes.Buffer{}
+	err = pprof.Lookup("goroutine").WriteTo(buf, 1)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// printStack prints the current goroutine stack to stderr.
+func printStack(_ *Request) (msg string, err error) {
+	_, err = fmt.Fprint(os.Stderr, "===== PRINTING STACK =====\n")
+	if err == nil {
+		err = pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
+	}
+	if err == nil {
+		_, err = fmt.Fprint(os.Stderr, "===== END OF STACK =====\n")
+	}
+	if err != nil {
+		return "", err
+	}
+	return "stack printed to stdout", nil
+}
+
+// debugInfo returns the debugging information for support requests.
+func debugInfo(ar *Request) (data []byte, err error) {
+	// Create debug information helper.
+	di := new(osdetail.DebugInfo)
+	di.Style = ar.Request.URL.Query().Get("style")
+
+	// Add debug information.
+	di.AddVersionInfo()
+	di.AddPlatformInfo(ar.Context())
+	di.AddLastReportedModuleError()
+	di.AddLastUnexpectedLogs()
+	di.AddGoroutineStack()
+
+	// Return data.
+	return di.Bytes(), nil
+}

api/endpoints_meta.go

@@ -0,0 +1,65 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+func registerMetaEndpoints() error {
+	if err := RegisterEndpoint(Endpoint{
+		Path:     "endpoints",
+		Read:     PermitAnyone,
+		MimeType: MimeTypeJSON,
+		DataFunc: listEndpoints,
+	}); err != nil {
+		return err
+	}
+
+	if err := RegisterEndpoint(Endpoint{
+		Path:       "permission",
+		Read:       Require,
+		StructFunc: permissions,
+	}); err != nil {
+		return err
+	}
+
+	if err := RegisterEndpoint(Endpoint{
+		Path:       "ping",
+		Read:       PermitAnyone,
+		ActionFunc: ping,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func listEndpoints(ar *Request) (data []byte, err error) {
+	endpointsLock.Lock()
+	defer endpointsLock.Unlock()
+
+	data, err = json.Marshal(endpoints)
+	return
+}
+
+func permissions(ar *Request) (i interface{}, err error) {
+	if ar.AuthToken == nil {
+		return nil, errors.New("authentication token missing")
+	}
+
+	return struct {
+		Read          Permission
+		Write         Permission
+		ReadPermName  string
+		WritePermName string
+	}{
+		Read:          ar.AuthToken.Read,
+		Write:         ar.AuthToken.Write,
+		ReadPermName:  ar.AuthToken.Read.String(),
+		WritePermName: ar.AuthToken.Write.String(),
+	}, nil
+}
+
+func ping(ar *Request) (msg string, err error) {
+	return "Pong.", nil
+}

api/endpoints_test.go

@@ -0,0 +1,156 @@
+package api
+
+import (
+	"errors"
+	"sync"
+	"testing"
+
+	"github.com/safing/portbase/database/record"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	successMsg = "endpoint api success"
+	failedMsg  = "endpoint api failed"
+)
+
+type actionTestRecord struct {
+	record.Base
+	sync.Mutex
+	Msg string
+}
+
+func TestEndpoints(t *testing.T) {
+	testHandler := &mainHandler{
+		mux: mainMux,
+	}
+
+	// ActionFn
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/action",
+		Read: PermitAnyone,
+		ActionFunc: func(_ *Request) (msg string, err error) {
+			return successMsg, nil
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/action", nil, successMsg)
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/action-err",
+		Read: PermitAnyone,
+		ActionFunc: func(_ *Request) (msg string, err error) {
+			return "", errors.New(failedMsg)
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/action-err", nil, failedMsg)
+
+	// DataFn
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/data",
+		Read: PermitAnyone,
+		DataFunc: func(_ *Request) (data []byte, err error) {
+			return []byte(successMsg), nil
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/data", nil, successMsg)
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/data-err",
+		Read: PermitAnyone,
+		DataFunc: func(_ *Request) (data []byte, err error) {
+			return nil, errors.New(failedMsg)
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/data-err", nil, failedMsg)
+
+	// StructFn
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/struct",
+		Read: PermitAnyone,
+		StructFunc: func(_ *Request) (i interface{}, err error) {
+			return &actionTestRecord{
+				Msg: successMsg,
+			}, nil
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/struct", nil, successMsg)
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/struct-err",
+		Read: PermitAnyone,
+		StructFunc: func(_ *Request) (i interface{}, err error) {
+			return nil, errors.New(failedMsg)
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/struct-err", nil, failedMsg)
+
+	// RecordFn
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/record",
+		Read: PermitAnyone,
+		RecordFunc: func(_ *Request) (r record.Record, err error) {
+			r = &actionTestRecord{
+				Msg: successMsg,
+			}
+			r.CreateMeta()
+			return r, nil
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/record", nil, successMsg)
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/record-err",
+		Read: PermitAnyone,
+		RecordFunc: func(_ *Request) (r record.Record, err error) {
+			return nil, errors.New(failedMsg)
+		},
+	}))
+	assert.HTTPBodyContains(t, testHandler.ServeHTTP, "GET", apiV1Path+"test/record-err", nil, failedMsg)
+}
+
+func TestActionRegistration(t *testing.T) {
+	assert.Error(t, RegisterEndpoint(Endpoint{}))
+
+	assert.Error(t, RegisterEndpoint(Endpoint{
+		Path: "test/err",
+		Read: NotFound,
+	}))
+	assert.Error(t, RegisterEndpoint(Endpoint{
+		Path: "test/err",
+		Read: PermitSelf + 1,
+	}))
+
+	assert.Error(t, RegisterEndpoint(Endpoint{
+		Path:  "test/err",
+		Write: NotFound,
+	}))
+	assert.Error(t, RegisterEndpoint(Endpoint{
+		Path:  "test/err",
+		Write: PermitSelf + 1,
+	}))
+
+	assert.Error(t, RegisterEndpoint(Endpoint{
+		Path: "test/err",
+	}))
+
+	assert.Error(t, RegisterEndpoint(Endpoint{
+		Path: "test/err",
+		ActionFunc: func(_ *Request) (msg string, err error) {
+			return successMsg, nil
+		},
+		DataFunc: func(_ *Request) (data []byte, err error) {
+			return []byte(successMsg), nil
+		},
+	}))
+
+	assert.NoError(t, RegisterEndpoint(Endpoint{
+		Path: "test/err",
+		ActionFunc: func(_ *Request) (msg string, err error) {
+			return successMsg, nil
+		},
+	}))
+}

api/main.go

@@ -26,7 +26,16 @@ func prep() error {
 	if getDefaultListenAddress() == "" {
 		return errors.New("no default listen address for api available")
 	}
-	return registerConfig()
+
+	if err := registerConfig(); err != nil {
+		return err
+	}
+
+	if err := registerDebugEndpoints(); err != nil {
+		return err
+	}
+
+	return registerMetaEndpoints()
 }
 
 func start() error {
@@ -34,10 +43,8 @@ func start() error {
 	go Serve()
 
 	// start api auth token cleaner
-	authFnLock.Lock()
-	defer authFnLock.Unlock()
-	if authFn != nil {
-		module.NewTask("clean api auth tokens", cleanAuthTokens).Repeat(time.Minute)
+	if authFnSet.IsSet() {
+		module.NewTask("clean api auth tokens", cleanAuthTokens).Repeat(5 * time.Minute)
 	}
 
 	return nil

api/main_test.go

@@ -0,0 +1,58 @@
+package api
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/safing/portbase/dataroot"
+	"github.com/safing/portbase/modules"
+
+	// API depends on the database for the database api.
+	_ "github.com/safing/portbase/database/dbmodule"
+)
+
+func init() {
+	defaultListenAddress = "127.0.0.1:8817"
+}
+
+func TestMain(m *testing.M) {
+	// enable module for testing
+	module.Enable()
+
+	// tmp dir for data root (db & config)
+	tmpDir, err := ioutil.TempDir("", "portbase-testing-")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create tmp dir: %s\n", err)
+		os.Exit(1)
+	}
+	// initialize data dir
+	err = dataroot.Initialize(tmpDir, 0755)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to initialize data root: %s\n", err)
+		os.Exit(1)
+	}
+
+	// start modules
+	var exitCode int
+	err = modules.Start()
+	if err != nil {
+		// starting failed
+		fmt.Fprintf(os.Stderr, "failed to setup test: %s\n", err)
+		exitCode = 1
+	} else {
+		// run tests
+		exitCode = m.Run()
+	}
+
+	// shutdown
+	_ = modules.Shutdown()
+	if modules.GetExitStatusCode() != 0 {
+		exitCode = modules.GetExitStatusCode()
+		fmt.Fprintf(os.Stderr, "failed to cleanly shutdown test: %s\n", err)
+	}
+	// clean up and exit
+	os.RemoveAll(tmpDir)
+	os.Exit(exitCode)
+}

api/request.go

@@ -0,0 +1,44 @@
+package api
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+)
+
+// Request is a support struct to pool more request related information.
+type Request struct {
+	// Request is the http request.
+	*http.Request
+
+	// InputData contains the request body for write operations.
+	InputData []byte
+
+	// Route of this request.
+	Route *mux.Route
+
+	// URLVars contains the URL variables extracted by the gorilla mux.
+	URLVars map[string]string
+
+	// AuthToken is the request-side authentication token assigned.
+	AuthToken *AuthToken
+
+	// HandlerCache can be used by handlers to cache data between handlers within a request.
+	HandlerCache interface{}
+}
+
+// apiRequestContextKey is a key used for the context key/value storage.
+type apiRequestContextKey struct{}
+
+var (
+	requestContextKey = apiRequestContextKey{}
+)
+
+// GetAPIRequest returns the API Request of the given http request.
+func GetAPIRequest(r *http.Request) *Request {
+	ar, ok := r.Context().Value(requestContextKey).(*Request)
+	if ok {
+		return ar
+	}
+	return nil
+}

api/router.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"sync"
 	"time"
@@ -56,7 +57,9 @@ func RegisterMiddleware(middleware Middleware) {
 func Serve() {
 	// configure server
 	server.Addr = listenAddressConfig()
-	server.Handler = middlewareHandler
+	server.Handler = &mainHandler{
+		mux: mainMux,
+	}
 
 	// start serving
 	log.Infof("api: starting to listen on %s", server.Addr)
@@ -76,7 +79,70 @@ func Serve() {
 	}
 }
 
-// GetMuxVars wraps github.com/gorilla/mux.Vars in order to mitigate context key issues in multi-repo projects.
-func GetMuxVars(r *http.Request) map[string]string {
-	return mux.Vars(r)
+type mainHandler struct {
+	mux *mux.Router
+}
+
+func (mh *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	_ = module.RunWorker("http request", func(_ context.Context) error {
+		return mh.handle(w, r)
+	})
+}
+
+func (mh *mainHandler) handle(w http.ResponseWriter, r *http.Request) error {
+	// Setup context trace logging.
+	ctx, tracer := log.AddTracer(r.Context())
+	lrw := NewLoggingResponseWriter(w, r)
+	// Add request context.
+	apiRequest := &Request{
+		Request: r,
+	}
+	ctx = context.WithValue(ctx, requestContextKey, apiRequest)
+	// Add context back to request.
+	r = r.WithContext(ctx)
+
+	tracer.Tracef("api request: %s ___ %s", r.RemoteAddr, r.RequestURI)
+	defer func() {
+		// Log request status.
+		if lrw.Status != 0 {
+			// If lrw.Status is 0, the request may have been hijacked.
+			tracer.Debugf("api request: %s %d %s", lrw.Request.RemoteAddr, lrw.Status, lrw.Request.RequestURI)
+		}
+		tracer.Submit()
+	}()
+
+	// Get handler for request.
+	// Gorilla does not support handling this on our own very well.
+	// See github.com/gorilla/mux.ServeHTTP for reference.
+	var match mux.RouteMatch
+	var handler http.Handler
+	if mh.mux.Match(r, &match) {
+		handler = match.Handler
+		apiRequest.Route = match.Route
+		apiRequest.URLVars = match.Vars
+	}
+
+	// Be sure that URLVars always is a map.
+	if apiRequest.URLVars == nil {
+		apiRequest.URLVars = make(map[string]string)
+	}
+
+	// Check authentication.
+	apiRequest.AuthToken = authenticateRequest(lrw, r, handler)
+	if apiRequest.AuthToken == nil {
+		// Authenticator already replied.
+		return nil
+	}
+
+	// Handle request.
+	switch {
+	case handler != nil:
+		handler.ServeHTTP(lrw, r)
+	case errors.Is(match.MatchErr, mux.ErrMethodMismatch):
+		http.Error(lrw, "Method not allowed.", http.StatusMethodNotAllowed)
+	default: // handler == nil or other error
+		http.Error(lrw, "Not found.", http.StatusNotFound)
+	}
+
+	return nil
 }

log/output.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"runtime/debug"
+	"sync"
 	"time"
 )
 
@@ -191,6 +192,8 @@ StackTrace:
 
 				// if currentLine and line are _not_ equal, output currentLine
 				adapter.Write(currentLine, duplicates)
+				// add to unexpected logs
+				addUnexpectedLogs(currentLine)
 				// reset duplicate counter
 				duplicates = 0
 				// set new currentLine
@@ -203,6 +206,8 @@ StackTrace:
 		// write final line
 		if currentLine != nil {
 			adapter.Write(currentLine, duplicates)
+			// add to unexpected logs
+			addUnexpectedLogs(currentLine)
 		}
 		// reset state
 		currentLine = nil //nolint:ineffassign
@@ -231,3 +236,60 @@ func finalizeWriting() {
 		}
 	}
 }
+
+// Last Unexpected Logs
+
+var (
+	lastUnexpectedLogs      = make([]string, 10)
+	lastUnexpectedLogsIndex int
+	lastUnexpectedLogsLock  sync.Mutex
+)
+
+func addUnexpectedLogs(line *logLine) {
+	// Add main line.
+	if line.level >= WarningLevel {
+		addUnexpectedLogLine(line)
+		return
+	}
+
+	// Check for unexpected lines in the tracer.
+	if line.tracer != nil {
+		for _, traceLine := range line.tracer.logs {
+			if traceLine.level >= WarningLevel {
+				// Add full trace.
+				addUnexpectedLogLine(line)
+				return
+			}
+		}
+	}
+}
+
+func addUnexpectedLogLine(line *logLine) {
+	lastUnexpectedLogsLock.Lock()
+	defer lastUnexpectedLogsLock.Unlock()
+
+	// Format line and add to logs.
+	lastUnexpectedLogs[lastUnexpectedLogsIndex] = formatLine(line, 0, false)
+
+	// Increase index and wrap back to start.
+	lastUnexpectedLogsIndex = (lastUnexpectedLogsIndex + 1) % len(lastUnexpectedLogs)
+}
+
+// GetLastUnexpectedLogs returns the last 10 log lines of level Warning an up.
+func GetLastUnexpectedLogs() []string {
+	lastUnexpectedLogsLock.Lock()
+	defer lastUnexpectedLogsLock.Unlock()
+
+	// Make a copy and return.
+	len := len(lastUnexpectedLogs)
+	start := lastUnexpectedLogsIndex
+	logsCopy := make([]string, 0, len)
+	// Loop from mid-to-mid.
+	for i := start; i < start+len; i++ {
+		if lastUnexpectedLogs[i%len] != "" {
+			logsCopy = append(logsCopy, lastUnexpectedLogs[i%len])
+		}
+	}
+
+	return logsCopy
+}

modules/error.go

@@ -11,7 +11,8 @@ import (
 var (
 	errorReportingChannel chan *ModuleError
 	reportToStdErr        = true
-	reportingLock         sync.RWMutex
+	lastReportedError     *ModuleError
+	reportingLock         sync.Mutex
 )
 
 // ModuleError wraps a panic, error or message into an error that can be reported.
@@ -67,10 +68,37 @@ func (me *ModuleError) Error() string {
 	return me.Message
 }
 
+// Format returns the error formatted in key/value form.
+func (me *ModuleError) Format() string {
+	return fmt.Sprintf(
+		`Message: %s
+Timestamp: %s
+ModuleName: %s
+TaskName: %s
+TaskType: %s
+Severity: %s
+PanicValue: %s
+StackTrace:
+
+%s
+`,
+		me.Message,
+		time.Now(),
+		me.ModuleName,
+		me.TaskName,
+		me.TaskType,
+		me.Severity,
+		me.PanicValue,
+		me.StackTrace,
+	)
+}
+
 // Report reports the error through the configured reporting channel.
 func (me *ModuleError) Report() {
-	reportingLock.RLock()
-	defer reportingLock.RUnlock()
+	reportingLock.Lock()
+	defer reportingLock.Unlock()
+
+	lastReportedError = me
 
 	if errorReportingChannel != nil {
 		select {
@@ -83,27 +111,8 @@ func (me *ModuleError) Report() {
 		// default to writing to stderr
 		fmt.Fprintf(
 			os.Stderr,
-			`===== Error Report =====
-Message: %s
-Timestamp: %s
-ModuleName: %s
-TaskName: %s
-TaskType: %s
-Severity: %s
-PanicValue: %s
-StackTrace:
-
-%s
-===== End of Report =====
-`,
-			me.Message,
-			time.Now(),
-			me.ModuleName,
-			me.TaskName,
-			me.TaskType,
-			me.Severity,
-			me.PanicValue,
-			me.StackTrace,
+			"===== Error Report =====\n%s\n===== End of Report =====\n",
+			me.Format(),
 		)
 	}
 }
@@ -133,3 +142,11 @@ func SetStdErrReporting(on bool) {
 
 	reportToStdErr = on
 }
+
+// GetLastReportedError returns the last reported module error.
+func GetLastReportedError() *ModuleError {
+	reportingLock.Lock()
+	defer reportingLock.Unlock()
+
+	return lastReportedError
+}

utils/osdetail/debug.go

@@ -0,0 +1,183 @@
+package osdetail
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"runtime/pprof"
+	"time"
+
+	"github.com/safing/portbase/log"
+
+	"github.com/safing/portbase/info"
+	"github.com/safing/portbase/modules"
+	"github.com/shirou/gopsutil/host"
+)
+
+// DebugInfo gathers debugging information and stores everything in a buffer in
+// order to write it to somewhere later. It directly inherits a bytes.Buffer,
+// so you can also use all these functions too.
+type DebugInfo struct {
+	bytes.Buffer
+	Style string
+}
+
+// DebugInfoFlag defines possible options for adding sections to a DebugInfo.
+type DebugInfoFlag int
+
+const (
+	// NoFlags does nothing.
+	NoFlags DebugInfoFlag = 0
+
+	// UseCodeSection wraps the section content in a markdown code section.
+	UseCodeSection DebugInfoFlag = 1
+
+	// AddContentLineBreaks adds a line breaks after each line of content,
+	// except for the last.
+	AddContentLineBreaks DebugInfoFlag = 2
+)
+
+func useCodeSection(flags DebugInfoFlag) bool {
+	return flags&UseCodeSection > 0
+}
+
+func addContentLineBreaks(flags DebugInfoFlag) bool {
+	return flags&AddContentLineBreaks > 0
+}
+
+// AddSection adds a debug section to the DebugInfo. The result is directly
+// written into the buffer.
+func (di *DebugInfo) AddSection(name string, flags DebugInfoFlag, content ...string) {
+	// Check if we need a spacer.
+	if di.Len() > 0 {
+		di.WriteString("\n\n")
+	}
+
+	// Write section to buffer.
+
+	// Write section header.
+	if di.Style == "github" {
+		di.WriteString(fmt.Sprintf("<details>\n<summary>%s</summary>\n\n", name))
+	} else {
+		di.WriteString(fmt.Sprintf("**%s**:\n\n", name))
+	}
+
+	// Write section content.
+	if useCodeSection(flags) {
+		// Write code header: Needs one empty line between previous data.
+		di.WriteString("```\n")
+	}
+	for i, part := range content {
+		di.WriteString(part)
+		if addContentLineBreaks(flags) && i < len(content)-1 {
+			di.WriteString("\n")
+		}
+	}
+	if useCodeSection(flags) {
+		// Write code footer: Needs one empty line between next data.
+		di.WriteString("\n```\n")
+	}
+
+	// Write section header.
+	if di.Style == "github" {
+		di.WriteString("\n</details>")
+	}
+}
+
+// AddVersionInfo adds version information from the info pkg.
+func (di *DebugInfo) AddVersionInfo() {
+	di.AddSection(
+		"Version "+info.Version(),
+		UseCodeSection,
+		info.FullVersion(),
+	)
+}
+
+// AddPlatformInfo adds OS and platform information.
+func (di *DebugInfo) AddPlatformInfo(ctx context.Context) {
+	// Get information from the system.
+	info, err := host.InfoWithContext(ctx)
+	if err != nil {
+		di.AddSection(
+			"Platform Information",
+			NoFlags,
+			fmt.Sprintf("Failed to get: %s", err),
+		)
+		return
+	}
+
+	// Check if we want to add virtulization information.
+	var virtInfo string
+	if info.VirtualizationRole == "guest" {
+		if info.VirtualizationSystem != "" {
+			virtInfo = fmt.Sprintf("VM: %s", info.VirtualizationSystem)
+		} else {
+			virtInfo = "VM: unidentified"
+		}
+	}
+
+	// Add section.
+	di.AddSection(
+		fmt.Sprintf("Platform: %s %s", info.Platform, info.PlatformVersion),
+		UseCodeSection|AddContentLineBreaks,
+		fmt.Sprintf("System: %s %s (%s) %s", info.Platform, info.OS, info.PlatformFamily, info.PlatformVersion),
+		fmt.Sprintf("Kernel: %s %s", info.KernelVersion, info.KernelArch),
+		virtInfo,
+	)
+}
+
+// AddGoroutineStack adds the current goroutine stack.
+func (di *DebugInfo) AddGoroutineStack() {
+	buf := new(bytes.Buffer)
+	err := pprof.Lookup("goroutine").WriteTo(buf, 1)
+	if err != nil {
+		di.AddSection(
+			"Goroutine Stack",
+			NoFlags,
+			fmt.Sprintf("Failed to get: %s", err),
+		)
+		return
+	}
+
+	// Add section.
+	di.AddSection(
+		"Goroutine Stack",
+		UseCodeSection,
+		buf.String(),
+	)
+}
+
+// AddLastReportedModuleError adds the last reported module error, if one exists.
+func (di *DebugInfo) AddLastReportedModuleError() {
+	me := modules.GetLastReportedError()
+	if me == nil {
+		di.AddSection("No Module Error", NoFlags)
+		return
+	}
+
+	di.AddSection(
+		"Module Error",
+		UseCodeSection,
+		me.Format(),
+	)
+}
+
+// AddLastUnexpectedLogs adds the last 10 unexpected log lines, if any.
+func (di *DebugInfo) AddLastUnexpectedLogs() {
+	lines := log.GetLastUnexpectedLogs()
+
+	// Check if there is anything at all.
+	if len(lines) == 0 {
+		di.AddSection("No Unexpected Logs", NoFlags)
+		return
+	}
+
+	di.AddSection(
+		"Unexpected Logs",
+		UseCodeSection|AddContentLineBreaks,
+		append(
+			lines,
+			fmt.Sprintf("%s CURRENT TIME", time.Now().Format("060102 15:04:05.000")),
+		)...,
+	)
+}